/* * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "core/core.h" #include "gpu/gpu.h" #include "nverror.h" #include "gpu/bif/kernel_bif.h" #include "gpu/fsp/kern_fsp.h" #include "platform/chipset/chipset.h" #include "ctrl/ctrl2080/ctrl2080bus.h" #include "published/hopper/gh100/dev_xtl_ep_pcfg_gpu.h" #include "published/hopper/gh100/dev_fb.h" #include "published/hopper/gh100/hwproject.h" #include "published/hopper/gh100/dev_xtl_ep_pri.h" #include "published/hopper/gh100/dev_nv_xpl.h" #include "published/hopper/gh100/dev_vm.h" #include "published/hopper/gh100/dev_pmc.h" #include "os/os.h" static NV_STATUS _kbifSavePcieConfigRegisters_GH100(OBJGPU *pGpu, KernelBif *pKernelBif, const PKBIF_XVE_REGMAP_REF pRegmapRef); static NV_STATUS _kbifRestorePcieConfigRegisters_GH100(OBJGPU *pGpu, KernelBif *pKernelBif, const PKBIF_XVE_REGMAP_REF pRegmapRef); // Size of PCIe config space #define PCIE_CONFIG_SPACE_MAX_SIZE 0x2FC // Devinit completion timeout after FLR #define BIF_FLR_DEVINIT_COMPLETION_TIMEOUT_DEFAULT 900000 /*! * @brief Check if MSI is enabled in HW * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * * @return True if MSI enabled else False */ NvBool kbifIsMSIEnabledInHW_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 data32; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_MSI_64_HEADER, &data32) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "unable to read NV_EP_PCFG_GPU_MSI_64_HEADER\n"); } return FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _MSI_64_HEADER, _MSI_ENABLE, 0x1, data32); } /*! * @brief Check if MSIX is enabled in HW * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * * @return True if MSIX enabled else False */ NvBool kbifIsMSIXEnabledInHW_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 data32; if (IS_VIRTUAL(pGpu)) { // SR-IOV guests only support MSI-X return IS_VIRTUAL_WITH_SRIOV(pGpu); } else { if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_MSIX_CAP_HEADER, &data32) != NV_OK) { NV_ASSERT_FAILED("Unable to read NV_EP_PCFG_GPU_MSIX_CAP_HEADER\n"); return NV_FALSE; } return FLD_TEST_DRF(_EP_PCFG_GPU, _MSIX_CAP_HEADER, _ENABLE, _ENABLED, data32); } } /*! * @brief Check if access to PCI config space is enabled or not * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * * @return NV_TRUE Pci IO access is enabled */ NvBool kbifIsPciIoAccessEnabled_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 data = 0; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_CTRL_CMD_AND_STATUS, &data) == NV_OK) { if (FLD_TEST_DRF(_EP_PCFG_GPU, _CTRL_CMD_AND_STATUS, _CMD_IO_SPACE, _ENABLE, data)) { return NV_TRUE; } } return NV_FALSE; } /*! * @brief Check if device is a 3D controller * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * * @return NV_TRUE If device is a 3D controller */ NvBool kbifIs3dController_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 data = 0; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_REVISION_ID_AND_CLASSCODE, &data) == NV_OK) { if (FLD_TEST_DRF(_EP_PCFG_GPU, _REVISION_ID_AND_CLASSCODE, _BASE_CLASSCODE, _3D, data)) { return NV_TRUE; } } return NV_FALSE; } /*! * @brief Enables extended tag support for GPU. * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * */ void kbifEnableExtendedTagSupport_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 reg; OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CAPABILITIES, ®) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CAPABILITIES\n"); goto _kbifEnableExtendedTagSupport_GH100_exit; } reg = GPU_DRF_VAL(_EP_PCFG_GPU, _DEVICE_CAPABILITIES, _EXTENDED_TAG_FIELD_SUPPORTED, reg); if ((reg != 0) && !pCl->getProperty(pCl, PDB_PROP_CL_EXTENDED_TAG_FIELD_NOT_CAPABLE)) { if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, ®) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS\n"); goto _kbifEnableExtendedTagSupport_GH100_exit; } reg = FLD_SET_DRF(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _EXTENDED_TAG_FIELD_ENABLE, _INIT, reg); if (GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, reg) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to write NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS\n"); goto _kbifEnableExtendedTagSupport_GH100_exit; } } _kbifEnableExtendedTagSupport_GH100_exit: return; } /*! * @brief Enable/disable no snoop for GPU * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * @param[in] bEnable True if No snoop needs to be enabled * * @return NV_OK If no snoop modified as requested */ NV_STATUS kbifEnableNoSnoop_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvBool bEnable ) { NvU8 fieldVal; NvU32 regVal; NvU32 status = NV_OK; status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, ®Val); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Failed to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS.\n"); goto _kbifEnableNoSnoop_GH100_exit; } fieldVal = bEnable ? 1 : 0; regVal = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _ENABLE_NO_SNOOP, fieldVal, regVal); status = GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, regVal); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Failed to write NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS.\n"); goto _kbifEnableNoSnoop_GH100_exit; } _kbifEnableNoSnoop_GH100_exit: return status; } /*! * @brief Enables Relaxed Ordering PCI-E Capability in the PCI Config Space * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer */ void kbifPcieConfigEnableRelaxedOrdering_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 xveDevCtrlStatus; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, &xveDevCtrlStatus) == NV_ERR_GENERIC) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS!\n"); } else { GPU_BUS_CFG_CYCLE_FLD_WR_DRF_DEF(pGpu, xveDevCtrlStatus, _EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _ENABLE_RELAXED_ORDERING, _INIT); } } /*! * @brief Disables Relaxed Ordering PCI-E Capability in the PCI Config Space * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer */ void kbifPcieConfigDisableRelaxedOrdering_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 xtlDevCtrlStatus; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, &xtlDevCtrlStatus) == NV_ERR_GENERIC) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS!\n"); } else { xtlDevCtrlStatus = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _ENABLE_RELAXED_ORDERING, 0, xtlDevCtrlStatus); GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, xtlDevCtrlStatus); } } /*! * @brief Get the status of XTL * (Function name ideally should be bifGetXtlStatusBits) * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif BIF object pointer * @param[out] pBits PCIE error status values * @param[out] pStatus Full XTL status * * @return NV_OK */ NV_STATUS kbifGetXveStatusBits_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvU32 *pBits, NvU32 *pStatus ) { // control/status reg 0x68 NvU32 xtlDevCtrlStatus; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, &xtlDevCtrlStatus) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS!\n"); } if (pBits == NULL) return NV_ERR_GENERIC; *pBits = 0; // The register read above returns garbage on fmodel, so just return. if (IS_FMODEL(pGpu)) { if (pStatus) { *pStatus = 0; } return NV_OK; } // Optionally return full status if (pStatus) { *pStatus = xtlDevCtrlStatus; } if (xtlDevCtrlStatus & DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _CORR_ERROR_DETECTED, 1)) { *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_CORR_ERROR; } if (xtlDevCtrlStatus & DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _NON_FATAL_ERROR_DETECTED, 1)) { *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_NON_FATAL_ERROR; } if (xtlDevCtrlStatus & DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _FATAL_ERROR_DETECTED, 1)) { *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_FATAL_ERROR; } if (xtlDevCtrlStatus & DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _UNSUPP_REQUEST_DETECTED, 1)) { *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_UNSUPP_REQUEST; } if (pKernelBif->EnteredRecoverySinceErrorsLastChecked) { pKernelBif->EnteredRecoverySinceErrorsLastChecked = NV_FALSE; *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_ERRORS_ENTERED_RECOVERY; } return NV_OK; } /*! * @brief Clear XVE status * (Function name ideally should be bifClearXtlStatus) * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif BIF object pointer * @param[out] pStatus Full XTL status * * @return NV_OK */ NV_STATUS kbifClearXveStatus_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvU32 *pStatus ) { NvU32 xtlDevCtrlStatus; if (pStatus) { xtlDevCtrlStatus = *pStatus; if (xtlDevCtrlStatus == 0) { return NV_OK; } } else { if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, &xtlDevCtrlStatus) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS!\n"); } } GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, xtlDevCtrlStatus); return NV_OK; } /*! * @brief Get XTL AER bits * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif BIF object pointer * @param[out] pBits XTL AER bits value * * @return NV_OK */ NV_STATUS kbifGetXveAerBits_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvU32 *pBits ) { NvU32 xtlAerUncorr; NvU32 xtlAerCorr; if (pBits == NULL) return NV_ERR_GENERIC; // The register read below returns garbage on fmodel, so just return. if (IS_FMODEL(pGpu)) { return NV_OK; } *pBits = 0; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_UNCORRECTABLE_ERROR_STATUS, &xtlAerUncorr) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_UNCORRECTABLE_ERROR_STATUS\n"); return NV_ERR_GENERIC; } if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_CORRECTABLE_ERROR_STATUS, &xtlAerCorr) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_CORRECTABLE_ERROR_STATUS\n"); return NV_ERR_GENERIC; } if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _DL_PROTOCOL_ERROR, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_DLINK_PROTO_ERR; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _POISONED_TLP_RCVD, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_POISONED_TLP; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _COMPLETION_TIMEOUT, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_TIMEOUT; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _UNEXPECTED_COMPLETION, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNEXP_CPL; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _MALFORMED_TLP, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_MALFORMED_TLP; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _UNSUPPORTED_REQUEST_ERROR, 0x1, xtlAerUncorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNSUPPORTED_REQ; // FERMI-TODO if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _RECEIVER_ERROR, 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RCV_ERR; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _BAD_TLP, 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_TLP; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _BAD_DLLP , 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_DLLP; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _REPLAY_NUM_ROLLOVER, 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_ROLLOVER; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _REPLAY_TIMER_TIMEOUT, 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_TIMEOUT; if (FLD_TEST_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _ADVISORY_NON_FATAL_ERROR, 0x1, xtlAerCorr)) *pBits |= NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_ADVISORY_NONFATAL; return NV_OK; } /*! * @brief Clear XTL AER bits * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif BIF object pointer * @param[in] bits XTL AER bits to be cleared * * @return NV_OK */ NV_STATUS kbifClearXveAer_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvU32 bits ) { NvU32 xtlAerUncorr = 0; NvU32 xtlAerCorr = 0; if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_DLINK_PROTO_ERR) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _DL_PROTOCOL_ERROR, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_POISONED_TLP) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _POISONED_TLP_RCVD, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_CPL_TIMEOUT) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _COMPLETION_TIMEOUT, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNEXP_CPL) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _UNEXPECTED_COMPLETION, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_MALFORMED_TLP) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _MALFORMED_TLP, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_UNCORR_UNSUPPORTED_REQ) xtlAerUncorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _UNCORRECTABLE_ERROR_STATUS, _UNSUPPORTED_REQUEST_ERROR, 0x1, xtlAerUncorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RCV_ERR) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _RECEIVER_ERROR, 0x1, xtlAerCorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_TLP) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _BAD_TLP, 0x1, xtlAerCorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_BAD_DLLP) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _BAD_DLLP, 0x1, xtlAerCorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_ROLLOVER) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _REPLAY_NUM_ROLLOVER, 0x1, xtlAerCorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_RPLY_TIMEOUT) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _REPLAY_TIMER_TIMEOUT, 0x1, xtlAerCorr); if (bits & NV2080_CTRL_BUS_INFO_PCIE_LINK_AER_CORR_ADVISORY_NONFATAL) xtlAerCorr = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _CORRECTABLE_ERROR_STATUS, _ADVISORY_NON_FATAL_ERROR, 0x1, xtlAerCorr); if (xtlAerUncorr != 0) { GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_UNCORRECTABLE_ERROR_STATUS, xtlAerUncorr); } if (xtlAerCorr != 0) { GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_CORRECTABLE_ERROR_STATUS, xtlAerCorr); } return NV_OK; } /*! * @brief Return the BAR0 offset and size of the PCI config space mirror. * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * @param[out] pBase BAR0 offset of the PCI config space mirror * @param[out] pSize Size in bytes of the PCI config space mirror * * @returns NV_OK */ NV_STATUS kbifGetPciConfigSpacePriMirror_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvU32 *pBase, NvU32 *pSize ) { *pBase = DEVICE_BASE(NV_EP_PCFGM); *pSize = DEVICE_EXTENT(NV_EP_PCFGM) - DEVICE_BASE(NV_EP_PCFGM) + 1; return NV_OK; } /*! * Kernel-RM only function to enable PCIe requester atomics by * using OS HAL interface and to cache the completer side capabilities returned * by the OS HAL interface. RM regkeys "RMSysmemSelectAtomicsConfig" and * "RMSysmemOverridePcieReqAtomicOps" can be used to force override the * settings. * * * @param[in] pGpu OBJGPU pointer * @param[in/out] pKernelBif Kernel BIF object pointer, PCIe requester atomics details * are cached in here. * @returns None */ void kbifProbePcieReqAtomicCaps_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 osAtomicsMask = 0; OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); if (!pCl->getProperty(pCl, PDB_PROP_CL_BUG_3562968_WAR_ALLOW_PCIE_ATOMICS)) { NV_PRINTF(LEVEL_INFO, "PCIe atomics not supported in this platform!\n"); return; } if (osConfigurePcieReqAtomics(pGpu->pOsGpuInfo, &osAtomicsMask) != NV_OK || osAtomicsMask == 0) { NV_PRINTF(LEVEL_INFO, "PCIe requester atomics not enabled since " "completer is not capable!\n"); return; } pKernelBif->osPcieAtomicsOpMask = osAtomicsMask; // Program PCIe atomics register settings kbifEnablePcieAtomics_HAL(pGpu, pKernelBif); return; } /*! * @brief Enable PCIe atomics if PCIe hierarchy supports it * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * */ void kbifEnablePcieAtomics_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 regVal; if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS_2, ®Val) != NV_OK) { NV_PRINTF(LEVEL_INFO, "Read of NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS_2 failed.\n"); return; } regVal = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS_2, _ATOMIC_OP_REQUESTER_ENABLE, 0x1, regVal); if (GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS_2, regVal) != NV_OK) { NV_PRINTF(LEVEL_INFO, "Write of NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS_2 failed.\n"); return; } NV_PRINTF(LEVEL_INFO, "PCIe Requester atomics enabled.\n"); } /*! * @brief Get bus options: link specific capabilities or * pcie device specific parameters or pcie link specific parameters * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif BIF object pointer * @param[in] options XTL bus options * @param[out] pAddrReg Address of the register for the given bus option * * @return NV_OK on success * NV_ERR_GENERIC */ NV_STATUS kbifGetBusOptionsAddr_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, BUS_OPTIONS options, NvU32 *pAddrReg ) { NV_STATUS status = NV_OK; switch (options) { case BUS_OPTIONS_DEV_CONTROL_STATUS: *pAddrReg = NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS; break; case BUS_OPTIONS_DEV_CONTROL_STATUS_2: *pAddrReg = NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS_2; break; case BUS_OPTIONS_LINK_CONTROL_STATUS: *pAddrReg = NV_EP_PCFG_GPU_LINK_CONTROL_STATUS; break; case BUS_OPTIONS_LINK_CAPABILITIES: *pAddrReg = NV_EP_PCFG_GPU_LINK_CAPABILITIES; break; case BUS_OPTIONS_L1_PM_SUBSTATES_CTRL_1: *pAddrReg = NV_EP_PCFG_GPU_L1_PM_SS_CONTROL_1_REGISTER; break; default: NV_PRINTF(LEVEL_ERROR, "Invalid register type passed 0x%x\n", options); status = NV_ERR_GENERIC; break; } return status; } /*! * @brief Helper function for _kbifSavePcieConfigRegisters_GH100() * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * @param[in] pRegmapRef Pointer to XVE Register map structure * * @return 'NV_OK' if successful, an RM error code otherwise. */ static NV_STATUS _kbifSavePcieConfigRegisters_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, const PKBIF_XVE_REGMAP_REF pRegmapRef ) { NV_STATUS status; NvU32 regOffset; NvU32 bufOffset = 0; // Read and save entire config space for (regOffset = 0x0; regOffset < PCIE_CONFIG_SPACE_MAX_SIZE; regOffset+=0x4) { status = GPU_BUS_CFG_CYCLE_RD32(pGpu, regOffset, &pRegmapRef->bufBootConfigSpace[bufOffset]); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Config read failed.\n"); return status; } bufOffset++; } pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_SUPPORTED, NV_TRUE); return NV_OK; } /*! * @brief Helper function for _kbifRestorePcieConfigRegisters_GH100 * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * @param[in] pRegmapRef Pointer to XVE Register map structure * * @return 'NV_OK' if successful, an RM error code otherwise. */ static NV_STATUS _kbifRestorePcieConfigRegisters_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, const PKBIF_XVE_REGMAP_REF pRegmapRef ) { NvU32 domain = gpuGetDomain(pGpu); NvU8 bus = gpuGetBus(pGpu); NvU8 device = gpuGetDevice(pGpu); void *pHandle; NvU32 regOffset; NvU32 bufOffset = 0; NV_STATUS status; pHandle = osPciInitHandle(domain, bus, device, 0, NULL, NULL); NV_ASSERT_OR_RETURN(pHandle, NV_ERR_INVALID_POINTER); // Restore entire config space for (regOffset = 0x0; regOffset < PCIE_CONFIG_SPACE_MAX_SIZE; regOffset+=0x4) { status = GPU_BUS_CFG_CYCLE_WR32(pGpu, regOffset, pRegmapRef->bufBootConfigSpace[bufOffset]); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Config write failed.\n"); NV_ASSERT(0); return status; } bufOffset++; } return NV_OK; } /*! * @brief Helper function saves MSIX vector control masks * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return 'NV_OK' if successful, an RM error code otherwise. */ NV_STATUS kbifSaveMsixTable_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 i; NvU32 controlSize = kbifGetMSIXTableVectorControlSize_HAL(pGpu, pKernelBif); for (i = 0U; i < controlSize; i++) { // Each MSIX table entry is 4 NvU32s pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 0] = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_LO(i)); pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 1] = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_HI(i)); pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 2] = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_DATA(i)); pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 3] = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL(i)); } return NV_OK; } /*! * @brief Helper function to restore MSIX vector control masks * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return 'NV_OK' if successful, an RM error code otherwise. */ NV_STATUS kbifRestoreMsixTable_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 i; NvU32 controlSize = kbifGetMSIXTableVectorControlSize_HAL(pGpu, pKernelBif); // Initialize the base offset for the virtual registers for physical function NvU32 vRegOffset = pGpu->sriovState.virtualRegPhysOffset; // In FLR path, we don't want to use usual register r/w macros for (i = 0U; i < controlSize; i++) { // Each MSIX table entry is 4 NvU32s osGpuWriteReg032(pGpu, vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_LO(i), pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 0]); osGpuWriteReg032(pGpu, vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_ADDR_HI(i), pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 1]); osGpuWriteReg032(pGpu, vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_DATA(i), pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 2]); osGpuWriteReg032(pGpu, vRegOffset + NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL(i), pKernelBif->xveRegmapRef[0].bufMsixTable[(i*4) + 3]); } return NV_OK; } /*! * @brief Wait for to get config access. * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * @param[in] pTimeout Value in microseconds to dev init complete * * @return NV_OK if successful. */ NV_STATUS kbifConfigAccessWait_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, RMTIMEOUT *pTimeout ) { NvU32 regVal = 0; NV_STATUS status = NV_OK; while (NV_TRUE) { // // This read only register should be accessible over config cycle // once devinit is complete without RM having to restore config space. // This register is not really reset on FLR. // if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_ID, ®Val) == NV_OK) { if (FLD_TEST_DRF(_EP_PCFG_GPU, _ID, _VENDOR, _NVIDIA, regVal)) { break; } } status = gpuCheckTimeout(pGpu, pTimeout); if (status == NV_ERR_TIMEOUT) { NV_ASSERT_FAILED("Timed out waiting for devinit to complete\n"); return status; } // // Put ourself into wait state for 1ms. This function runs in the context // of DxgkDdiResetFromTimeout which is at PASSIVE_LEVEL(lower priority) // which means it can be in wait state for longer delays of the order of // milliseconds // osDelay(1); } return status; } /*! * @brief Do function level reset for Fn0 of GPU * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * * @return NV_OK if successful */ NV_STATUS kbifDoFunctionLevelReset_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvBool bMSIXEnabled; RMTIMEOUT timeout; NV_STATUS status = NV_OK; NvU32 flrDevInitTimeout; NvU32 flrDevInitTimeoutScale = pKernelBif->flrDevInitTimeoutScale; // If this is non-windows platform or non-ESXi, we already use OS based interface { pKernelBif->bInFunctionLevelReset = NV_TRUE; status = osDoFunctionLevelReset(pGpu); if (status != NV_OK) { NV_ASSERT_FAILED("osDoFunctionLevelReset failed!\n"); } goto kbifDoFunctionLevelReset_GH100_exit; } pKernelBif->bPreparingFunctionLevelReset = NV_TRUE; status = kbifSavePcieConfigRegisters_HAL(pGpu, pKernelBif); if (status != NV_OK) { NV_ASSERT_FAILED("Config registers save failed!\n"); goto kbifDoFunctionLevelReset_GH100_exit; } bMSIXEnabled = kbifIsMSIXEnabledInHW_HAL(pGpu, pKernelBif); if (bMSIXEnabled) { status = kbifSaveMsixTable_HAL(pGpu, pKernelBif); if (status != NV_OK) { NV_ASSERT_FAILED("MSIX Table save failed!\n"); goto kbifDoFunctionLevelReset_GH100_exit; } } // Once we save required registers, we are done with prep. phase for FLR pKernelBif->bPreparingFunctionLevelReset = NV_FALSE; // Trigger FLR now status = kbifTriggerFlr_HAL(pGpu, pKernelBif); if (status != NV_OK) { goto kbifDoFunctionLevelReset_GH100_exit; } pKernelBif->bInFunctionLevelReset = NV_TRUE; // // Wait for 10ms on Silicon for reset to propagate. Actually, HW is // supposed to return CRS status for config requests until config // space is not ready but we do expect at least 10ms for it to be ready // so why poll? On RTL and emulation, CPU clocks don't scale to GPU clock // so don't wait. // if (IS_SILICON(pGpu)) { osDelayUs(10000); } // flrTimeoutScale is regkey based for easy debugging flrDevInitTimeout = BIF_FLR_DEVINIT_COMPLETION_TIMEOUT_DEFAULT * flrDevInitTimeoutScale; // // After CRS_TIMEOUT, HW will auto-clear SEND_CRS bit which means config cycles // will succeed but it is still possible that devinit is not complete yet. // OS won't have a way to check that anyway but RM should do that here. // TODO: Should we use this loop or just the PCI spec-defined 100ms delay // now that we have a devinit wait in fspWaitForSecureBoot_HAL below? // gpuSetTimeout(pGpu, flrDevInitTimeout, &timeout, GPU_TIMEOUT_FLAGS_OSTIMER); status = kbifConfigAccessWait_HAL(pGpu, pKernelBif, &timeout); if (status != NV_OK) { NV_ASSERT_FAILED("Timed out waiting for devinit to complete\n"); goto kbifDoFunctionLevelReset_GH100_exit; } status = kbifRestorePcieConfigRegisters_HAL(pGpu, pKernelBif); if (status != NV_OK) { NV_ASSERT_FAILED("Config registers restore failed!\n"); goto kbifDoFunctionLevelReset_GH100_exit; } // As of GH100, miata rom cannot be run on simulation. if (IS_SILICON(pGpu) || IS_EMULATION(pGpu)) { // On emulation VBIOS boot can take long so add prints for better visibility if (IS_EMULATION(pGpu)) { NV_PRINTF(LEVEL_ERROR, "Entering secure boot completion wait.\n"); } status = NV_ERR_NOT_SUPPORTED; if (status != NV_OK) { DBG_BREAKPOINT(); NV_PRINTF(LEVEL_ERROR, "VBIOS boot failed!!\n"); goto kbifDoFunctionLevelReset_GH100_exit; } if (IS_EMULATION(pGpu)) { NV_PRINTF(LEVEL_ERROR, "Exited secure boot completion wait with status = NV_OK.\n"); } } if (bMSIXEnabled) { // TODO-NM: Check if this needed for other NVPM flows like GC6 status = kbifRestoreMsixTable_HAL(pGpu, pKernelBif); if (status != NV_OK) { NV_ASSERT_FAILED("MSIX Table save failed!\n"); } } kbifDoFunctionLevelReset_GH100_exit: pKernelBif->bPreparingFunctionLevelReset = NV_FALSE; pKernelBif->bInFunctionLevelReset = NV_FALSE; return status; } /*! * @brief Returns size of MSIX vector control table * * @param pGpu OBJGPU pointer * @param pKernelBif Kernel BIF object pointer */ NvU32 kbifGetMSIXTableVectorControlSize_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { return NV_VIRTUAL_FUNCTION_PRIV_MSIX_TABLE_VECTOR_CONTROL__SIZE_1; } /*! * @brief Check and cache Function level reset support * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * */ void kbifCacheFlrSupport_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 regVal = 0; // Read config register if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CAPABILITIES, ®Val) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_DEVICE_CAPABILITIES\n"); return; } // Check if FLR is supported if (FLD_TEST_DRF(_EP_PCFG_GPU, _DEVICE_CAPABILITIES, _FUNCTION_LEVEL_RESET_CAPABILITY, _SUPPORTED, regVal)) { pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_FLR_SUPPORTED, NV_TRUE); } } /*! * @brief Check and cache 64b BAR0 support * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer * */ void kbifCache64bBar0Support_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 regVal = 0; // Read config register if (GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_BARREG0, ®Val) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_BARREG0\n"); return; } // Check if 64b BAR0 is supported if (FLD_TEST_DRF(_EP_PCFG_GPU, _BARREG0, _REG_ADDR_TYPE, _64BIT, regVal)) { pKernelBif->setProperty(pKernelBif, PDB_PROP_KBIF_64BIT_BAR0_SUPPORTED, NV_TRUE); } } /*! * @brief: Get BAR information from PCIe config space * * @param[in] pGpu OBJGPU pointer * @param[in] barRegCSBase The base register 0 address * @param[in] barIndex The BAR index to check * @param[out] pBarBaseAddress The start address of the specified BAR * @param[out] pIs64BitBar To indicate if the BAR is using 64bit address * * @returns NV_STATUS */ static NV_STATUS _kbifGetBarInfo_GH100 ( OBJGPU *pGpu, NvU32 barRegCSBase, NvU32 barIndex, NvU64 *pBarBaseAddress, NvBool *pIs64BitBar ) { NV_STATUS status = NV_OK; NvBool barIs64Bit = NV_FALSE; NvU32 barAddrLow = 0; NvU32 barAddrHigh = 0; NvU32 barRegCSLimit = barRegCSBase + NV_EP_PCFG_GPU_BARREG5 - NV_EP_PCFG_GPU_BARREG0; NvU32 barRegCSOffset = barRegCSBase; NvU64 barBaseAddr = 0; NvU32 i = 0; for (i = 0; i <= barIndex; i++) { if ((status = GPU_BUS_CFG_CYCLE_RD32(pGpu, barRegCSOffset, &barAddrLow)) != NV_OK) { return status; } // // The SPACE_TYPE, ADDRESS_TYPE, PREFETCHABLE and BASE_ADDRESS fields // have the same definition as for Base Address Register 0 // barIs64Bit = FLD_TEST_DRF(_EP_PCFG_GPU, _BARREG0, _REG_ADDR_TYPE, _64BIT, barAddrLow); if (i != barIndex) { barRegCSOffset += (barIs64Bit ? 8 : 4); if (barRegCSOffset >= barRegCSLimit) { return NV_ERR_INVALID_INDEX; } } } if (pBarBaseAddress != NULL) { // Get the BAR address barBaseAddr = barAddrLow & 0xFFFFFFF0; if (barIs64Bit) { // Read and save the bar high address status = GPU_BUS_CFG_CYCLE_RD32(pGpu, barRegCSOffset + 4, &barAddrHigh); NV_ASSERT_OR_RETURN((status == NV_OK), status); barBaseAddr |= (NvU64)barAddrHigh << 32; } *pBarBaseAddress = barBaseAddr; } if (pIs64BitBar != NULL) { *pIs64BitBar = barIs64Bit; } return NV_OK; } /*! @brief Fetch VF details such as no. of VFs, First VF offset etc * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif Kernel BIF object pointer */ void kbifCacheVFInfo_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NV_STATUS status = NV_OK; NvU32 regVal = 0; NvU64 barAddr = 0; NvBool barIs64Bit = NV_FALSE; // Get total VF count status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_SRIOV_INIT_TOT_VF, ®Val); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_SRIOV_INIT_TOT_VF\n"); return; } pGpu->sriovState.totalVFs = GPU_DRF_VAL(_EP_PCFG_GPU, _SRIOV_INIT_TOT_VF, _TOTAL_VFS, regVal); // Get first VF offset status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_SRIOV_FIRST_VF_STRIDE, ®Val); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Unable to read NV_EP_PCFG_GPU_SRIOV_FIRST_VF_STRIDE\n"); return; } pGpu->sriovState.firstVFOffset = GPU_DRF_VAL(_EP_PCFG_GPU, _SRIOV_FIRST_VF_STRIDE, _FIRST_VF_OFFSET, regVal); // Get VF BAR0 info status = _kbifGetBarInfo_GH100(pGpu, NV_EP_PCFG_GPU_VF_BAR0, 0, &barAddr, &barIs64Bit); NV_ASSERT(status == NV_OK); pGpu->sriovState.firstVFBarAddress[0] = barAddr; pGpu->sriovState.b64bitVFBar0 = barIs64Bit; // Get VF BAR1 info status = _kbifGetBarInfo_GH100(pGpu, NV_EP_PCFG_GPU_VF_BAR0, 1, &barAddr, &barIs64Bit); NV_ASSERT(status == NV_OK); pGpu->sriovState.firstVFBarAddress[1] = barAddr; pGpu->sriovState.b64bitVFBar1 = barIs64Bit; // Get VF BAR2 info status = _kbifGetBarInfo_GH100(pGpu, NV_EP_PCFG_GPU_VF_BAR0, 2, &barAddr, &barIs64Bit); NV_ASSERT(status == NV_OK); pGpu->sriovState.firstVFBarAddress[2] = barAddr; pGpu->sriovState.b64bitVFBar2 = barIs64Bit; } /*! * @brief Clears Bus Master Enable bit in command register, disabling * Function 0 - from issuing any new requests to sysmem. * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return NV_OK */ NV_STATUS kbifStopSysMemRequests_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif, NvBool bStop ) { NvU32 regVal; NV_ASSERT_OK_OR_RETURN(GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_CTRL_CMD_AND_STATUS, ®Val)); if (bStop) { regVal = FLD_SET_DRF(_EP_PCFG_GPU, _CTRL_CMD_AND_STATUS, _CMD_BUS_MASTER, _DISABLE, regVal); } else { regVal = FLD_SET_DRF(_EP_PCFG_GPU, _CTRL_CMD_AND_STATUS, _CMD_BUS_MASTER, _ENABLE, regVal); } NV_ASSERT_OK_OR_RETURN(GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_CTRL_CMD_AND_STATUS, regVal)); return NV_OK; } /*! * @brief Save boot time PCIe Config space * * @param[in] pGpu GPU object pointer * @param[in] pBif BIF object pointer * * @return 'NV_OK' if successful, an RM error code otherwise. */ NV_STATUS kbifSavePcieConfigRegisters_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NV_STATUS status = NV_OK; // Skip config save on FMODEL (200684952), move to regmap (200700271) if (IS_FMODEL(pGpu)) { return NV_OK; } // // Save config space if GPU is about to enter Function Level Reset // OR if on non-windows platform, FORCE_PCIE_CONFIG_SAVE is set and SBR is // enabled // OR if on windows platform, SBR is enabled // if (!pKernelBif->bPreparingFunctionLevelReset && !((RMCFG_FEATURE_PLATFORM_WINDOWS || pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_FORCE_PCIE_CONFIG_SAVE)) && pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_SECONDARY_BUS_RESET_ENABLED))) { NV_PRINTF(LEVEL_ERROR, "Config space save has been skipped.\n"); return NV_OK; } // Save pcie config space for function 0 status = _kbifSavePcieConfigRegisters_GH100(pGpu, pKernelBif, &pKernelBif->xveRegmapRef[0]); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Saving PCIe config space failed for gpu.\n"); NV_ASSERT(0); return status; } return status; } /*! * @brief Restore boot time PCIe Config space * * @param[in] pGpu GPU object pointer * @param[in] pBif BIF object pointer * * @return 'NV_OK' if successful, an RM error code otherwise. */ NV_STATUS kbifRestorePcieConfigRegisters_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NV_STATUS status = NV_OK; // Skip config restore on FMODEL (200684952), move to regmap (200700271) if (IS_FMODEL(pGpu)) { return NV_OK; } // Restore pcie config space for function 0 status = _kbifRestorePcieConfigRegisters_GH100(pGpu, pKernelBif, &pKernelBif->xveRegmapRef[0]); if (status != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Restoring PCIe config space failed for gpu.\n"); NV_ASSERT(0); return status; } return status; } /*! * @brief Waits for function issued transaction completions (sysmem to GPU) to arrive * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return NV_OK */ NV_STATUS kbifWaitForTransactionsComplete_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { RMTIMEOUT timeout; NvU32 regVal; NV_ASSERT_OK_OR_RETURN(GPU_BUS_CFG_CYCLE_RD32( pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, ®Val)); gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0); // Wait for number of pending transactions to go to 0 while (DRF_VAL(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS, _TRANSACTIONS_PENDING, regVal) != 0) { NV_ASSERT_OK_OR_RETURN(gpuCheckTimeout(pGpu, &timeout)); NV_ASSERT_OK_OR_RETURN(GPU_BUS_CFG_CYCLE_RD32( pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, ®Val)); } return NV_OK; } /*! * @brief Trigger FLR. * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return NV_OK if successful. */ NV_STATUS kbifTriggerFlr_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 regVal = 0; NV_STATUS status = NV_OK; status = GPU_BUS_CFG_CYCLE_RD32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, ®Val); if (status != NV_OK) { NV_ASSERT_FAILED("Config space read of device control failed\n"); return status; } regVal = FLD_SET_DRF_NUM(_EP_PCFG_GPU, _DEVICE_CONTROL_STATUS , _INITIATE_FN_LVL_RST, 0x1, regVal); status = GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_DEVICE_CONTROL_STATUS, regVal); if (status != NV_OK) { NV_ASSERT_FAILED("FLR trigger failed\n"); return status; } return status; } /*! * @brief Try restoring BAR registers and command register using config cycles * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return NV_OK on success * NV_ERR_INVALID_READ if the register read returns unexpected value * NV_ERR_OBJECT_NOT_FOUND if the object is not found */ NV_STATUS kbifRestoreBarsAndCommand_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 *pBarRegOffsets = pKernelBif->barRegOffsets; NvU32 barOffsetEntry; // Restore all BAR registers for (barOffsetEntry = 0; barOffsetEntry < KBIF_NUM_BAR_OFFSET_ENTRIES; barOffsetEntry++) { if (pBarRegOffsets[barOffsetEntry] != KBIF_INVALID_BAR_REG_OFFSET) { GPU_BUS_CFG_CYCLE_WR32(pGpu, pBarRegOffsets[barOffsetEntry], pKernelBif->cacheData.gpuBootConfigSpace[pBarRegOffsets[barOffsetEntry]/sizeof(NvU32)]); } } // Restore Device Control register GPU_BUS_CFG_CYCLE_WR32(pGpu, NV_EP_PCFG_GPU_CTRL_CMD_AND_STATUS, pKernelBif->cacheData.gpuBootConfigSpace[NV_EP_PCFG_GPU_CTRL_CMD_AND_STATUS/sizeof(NvU32)]); if (GPU_REG_RD32(pGpu, NV_PMC_BOOT_0) != pGpu->chipId0) { return NV_ERR_INVALID_READ; } return NV_OK; } /*! * @brief HAL specific BIF software state initialization * * @param[in] pGpu GPU object pointer * @param[in] pKernelBif KernelBif object pointer * * @return NV_OK on success */ NV_STATUS kbifInit_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { // Cache the offsets of BAR registers into an array for subsequent use kbifStoreBarRegOffsets_HAL(pGpu, pKernelBif, NV_EP_PCFG_GPU_VF_BAR0); return NV_OK; } NvU32 kbifGetEccCounts_GH100 ( OBJGPU *pGpu, KernelBif *pKernelBif ) { NvU32 regVal; NvU32 count = 0; // PCIE RBUF regVal = GPU_REG_RD32(pGpu, NV_XPL_BASE_ADDRESS + NV_XPL_DL_ERR_COUNT_RBUF); count += DRF_VAL(_XPL_DL, _ERR_COUNT_RBUF, _UNCORR_ERR, regVal); // PCIE SEQ_LUT regVal = GPU_REG_RD32(pGpu, NV_XPL_BASE_ADDRESS + NV_XPL_DL_ERR_COUNT_SEQ_LUT); count += DRF_VAL(_XPL_DL, _ERR_COUNT_SEQ_LUT, _UNCORR_ERR, regVal); // PCIE XTL regVal = GPU_REG_RD32(pGpu, NV_XTL_BASE_ADDRESS + NV_XTL_EP_PRI_DED_ERROR_STATUS); if (regVal != 0) { count += 1; } // PCIE XTL regVal = GPU_REG_RD32(pGpu, NV_XTL_BASE_ADDRESS + NV_XTL_EP_PRI_RAM_ERROR_INTR_STATUS); if (regVal != 0) { count += 1; } return count; }