1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2018-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "ctrl/ctrl2080/ctrl2080ce.h"
25 #include "ctrl/ctrl2080/ctrl2080nvlink.h"
26 #include "gpu/ce/kernel_ce.h"
27 #include "gpu/nvlink/kernel_nvlink.h"
28 #include "gpu/ce/kernel_ce_private.h"
29 #include "gpu/gpu.h"
30
31 #include "published/ampere/ga102/dev_ce.h"
32
33 #define NV_CE_SYS_ALLOWED_LCE_MASK 0x0C
34 #define NV_CE_GRCE_ALLOWED_LCE_MASK 0x03
35 #define NV_CE_EVEN_ASYNC_LCE_MASK 0x00000010
36 #define NV_CE_ODD_ASYNC_LCE_MASK 0x00000000
37 #define NV_CE_MAX_LCE_MASK 0x1F
38 #define NV_CE_MAX_GRCE 2
39
40 /*!
41 * @brief Returns the size of the PCE2LCE register array
42 *
43 *
44 * @param[in] pGpu OBJGPU pointer
45 * @param[in] pKCe KernelCE pointer
46 *
47 * @return NV_CE_PCE2LCE_CONFIG__SIZE_1
48 *
49 */
50 NvU32
kceGetPce2lceConfigSize1_GA102(KernelCE * pKCe)51 kceGetPce2lceConfigSize1_GA102
52 (
53 KernelCE *pKCe
54 )
55 {
56 return NV_CE_PCE2LCE_CONFIG__SIZE_1;
57 }
58
59 /**
60 * @brief This function assigns PCE-LCE mappings for sysmem
61 * for the following two cases -
62 * 1. PCIe links - assign FBHUB PCEs
63 * 2. NVLinks - not POR for GA10X chips - not supported
64 * If sysLinkMask is 0, then we assume that sysmem is over PCIe.
65 * Else, follow step 2 as above.
66 *
67 * @param[in] pGpu OBJGPU pointer
68 * @param[in] pKCe KernelCE pointer
69 * @param[in] pceAvailableMaskPerHshub Pointer to CEs available per HSHUB
70 * @param[out] pLocalPceLceMap Pointer to PCE-LCE array
71 * @param[out] pLocalExposeCeMask Pointer to LCE Mask
72 *
73 * Returns NV_OK if successful in determining PCEs and LCEs for sysmem links
74 */
75 NV_STATUS
kceMapPceLceForSysmemLinks_GA102(OBJGPU * pGpu,KernelCE * pKCe,NvU32 * pceAvailableMaskPerHshub,NvU32 * pLocalPceLceMap,NvU32 * pLocalExposeCeMask,NvU32 fbPceMask)76 kceMapPceLceForSysmemLinks_GA102
77 (
78 OBJGPU *pGpu,
79 KernelCE *pKCe,
80 NvU32 *pceAvailableMaskPerHshub,
81 NvU32 *pLocalPceLceMap,
82 NvU32 *pLocalExposeCeMask,
83 NvU32 fbPceMask
84 )
85 {
86 NvU32 lceMask = 0;
87 NvU32 tempFbPceMask;
88 NvU32 lceIndex, pceIndex;
89 NV_STATUS status;
90
91 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
92
93 NV2080_CTRL_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK_PARAMS params;
94
95 NV_ASSERT_OR_RETURN(pKernelNvlink != NULL, NV_ERR_NOT_SUPPORTED);
96 portMemSet(¶ms, 0, sizeof(params));
97
98 status = knvlinkExecGspRmRpc(pGpu, pKernelNvlink,
99 NV2080_CTRL_CMD_NVLINK_HSHUB_GET_SYSMEM_NVLINK_MASK,
100 (void *)¶ms, sizeof(params));
101 if (status != NV_OK)
102 {
103 NV_PRINTF(LEVEL_ERROR, "Unable to determine PCEs and LCEs for sysmem links\n");
104 return status;
105 }
106
107 lceMask = kceGetSysmemSupportedLceMask_HAL(pGpu, pKCe);
108
109 //
110 // Assign FBHUB PCEs when sysmem is over PCIE because PCIE
111 // accesses are not supported over HSHUB PCEs
112 //
113 if (params.sysmemLinkMask == 0)
114 {
115 // Store lceMask in the exposeCeMask before moving on
116 *pLocalExposeCeMask |= lceMask;
117
118 tempFbPceMask = fbPceMask;
119 while(tempFbPceMask)
120 {
121 lceIndex = CE_GET_LOWEST_AVAILABLE_IDX(lceMask);
122 pceIndex = CE_GET_LOWEST_AVAILABLE_IDX(tempFbPceMask);
123 pLocalPceLceMap[pceIndex] = lceIndex;
124 // Clear the lowest set bits to get to the next index
125 tempFbPceMask &= (tempFbPceMask - 1);
126 lceMask &= (lceMask - 1);
127 }
128 }
129 else
130 {
131 // Print error message, do not assign PCEs and simply return
132 NV_PRINTF(LEVEL_ERROR,
133 "Sysmem over NVLink is not POR!\n");
134 }
135
136 return NV_OK;
137 }
138
139 /**
140 * @brief Returns mask of LCEs that can be assigned to sysmem connection
141 * where the index of corresponding set bit indicates the LCE index
142 *
143 * @param[in] pGpu OBJGPU pointer
144 * @param[in] pKCe KernelCE pointer
145 *
146 * Returns the mask of LCEs valid for SYSMEM connections
147 */
148 NvU32
kceGetSysmemSupportedLceMask_GA102(OBJGPU * pGpu,KernelCE * pKCe)149 kceGetSysmemSupportedLceMask_GA102
150 (
151 OBJGPU *pGpu,
152 KernelCE *pKCe
153 )
154 {
155 return (NV_CE_SYS_ALLOWED_LCE_MASK & NV_CE_MAX_LCE_MASK);
156 }
157
158 /**
159 * @brief Returns mask of LCEs that can be assigned to NVLink peers
160 * where the index of corresponding set bit indicates the LCE index
161 *
162 * @param[in] pGpu OBJGPU pointer
163 * @param[in] pKCe KernelCE pointer
164 *
165 * Returns the mask of LCEs valid for NVLink peers
166 */
167 NvU32
kceGetNvlinkPeerSupportedLceMask_GA102(OBJGPU * pGpu,KernelCE * pKCe,NvU32 peerAvailableLceMask)168 kceGetNvlinkPeerSupportedLceMask_GA102
169 (
170 OBJGPU *pGpu,
171 KernelCE *pKCe,
172 NvU32 peerAvailableLceMask
173 )
174 {
175 // There is no odd async LCE on GA10X - only LCE 4
176 return (NV_CE_EVEN_ASYNC_LCE_MASK & NV_CE_MAX_LCE_MASK);
177 }
178
179 /**
180 * @brief Returns mask of LCEs that can be assigned for GRCEs
181 * where the index of corresponding set bit indicates the LCE index
182 *
183 * @param[in] pGpu OBJGPU pointer
184 * @param[in] pKCe KernelCE pointer
185 *
186 * Returns the mask of LCEs valid for GRCEs
187 */
188 NvU32
kceGetGrceSupportedLceMask_GA102(OBJGPU * pGpu,KernelCE * pKCe)189 kceGetGrceSupportedLceMask_GA102
190 (
191 OBJGPU *pGpu,
192 KernelCE *pKCe
193 )
194 {
195 return (NV_CE_GRCE_ALLOWED_LCE_MASK & NV_CE_MAX_LCE_MASK);
196 }
197