1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "core/core.h"
25 #include "core/locks.h"
26 #include "gpu/subdevice/subdevice.h"
27 #include "mem_mgr/vaspace.h"
28 #include "mem_mgr/fabric_vaspace.h"
29 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
30 #include "kernel/gpu/nvlink/kernel_nvlink.h"
31 #include "gpu/bus/kern_bus.h"
32 
33 #include "vgpu/rpc.h"
34 #include "rmapi/client.h"
35 
36 static NV_STATUS
37 _subdeviceFlaRangeModeHostManagedVasDestroy
38 (
39     Subdevice *pSubdevice,
40     NV2080_CTRL_FLA_RANGE_PARAMS *pParams
41 )
42 {
43     OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
44     NvU32   gfid;
45 
46     NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
47     KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
48 
49     // See _subdeviceFlaRangeModeHostManagedVasInit for details about this check.
50     if (!RMCFG_FEATURE_PLATFORM_GSP && IS_GFID_PF(gfid) &&
51         (pKernelNvlink != NULL) && !knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink))
52         return NV_ERR_NOT_SUPPORTED;
53 
54     kbusDestroyHostManagedFlaVaspace_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), gfid);
55 
56     return NV_OK;
57 }
58 
59 static NV_STATUS
60 _subdeviceFlaRangeModeHostManagedVasInit
61 (
62     Subdevice *pSubdevice,
63     NV2080_CTRL_FLA_RANGE_PARAMS *pParams
64 )
65 {
66     OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
67     KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
68     NvU32 hSubdevice = RES_GET_HANDLE(pSubdevice);
69     NvHandle hDevice = RES_GET_PARENT_HANDLE(pSubdevice);
70     NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice);
71     NvU32 gfid;
72     NV_STATUS status;
73 
74     NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
75     KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
76     if (!RMCFG_FEATURE_PLATFORM_GSP && IS_GFID_PF(gfid) &&
77         (pKernelNvlink != NULL) && !knvlinkIsGpuConnectedToNvswitch(pGpu, pKernelNvlink))
78         return NV_ERR_NOT_SUPPORTED;
79 
80     NV_CHECK_OR_RETURN(LEVEL_SILENT, pParams->size != 0, NV_ERR_INVALID_ARGUMENT);
81 
82     if (pKernelBus->flaInfo.bFlaAllocated)
83     {
84         NV_PRINTF(LEVEL_ERROR,
85                   "FLA VAS is not allowed for base: %llx, size:%llx in gpu: %x\n",
86                   pParams->base, pParams->size, pGpu->gpuInstance);
87 
88         return NV_ERR_INVALID_STATE;
89     }
90 
91     status = kbusAllocateHostManagedFlaVaspace_HAL(pGpu, pKernelBus, hClient, hDevice,
92                                                   hSubdevice, pParams->hVASpace,
93                                                   pParams->base, pParams->size,
94                                                   gfid);
95     if (status != NV_OK)
96     {
97         NV_PRINTF(LEVEL_ERROR,
98                   "Host Managed FLA Vaspace failed, status: %x, gpu: %x \n",
99                   status, pGpu->gpuInstance);
100         return status;
101     }
102 
103     return NV_OK;
104 }
105 
106 static NV_STATUS
107 _subdeviceFlaRangeModeInit
108 (
109     Subdevice *pSubdevice,
110     NV2080_CTRL_FLA_RANGE_PARAMS *pParams
111 )
112 {
113     OBJGPU    *pGpu       = GPU_RES_GET_GPU(pSubdevice);
114     KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
115     NV_STATUS  status;
116 
117     KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
118 
119     // Must not be called from vGPU guests, except for arch MODS.
120     if (IS_VIRTUAL(pGpu))
121         return NV_ERR_NOT_SUPPORTED;
122 
123     NV_CHECK_OR_RETURN(LEVEL_SILENT, pParams->size != 0, NV_ERR_INVALID_ARGUMENT);
124 
125     //
126     // check if FM has previously allocated for the same FLA range.
127     //
128     // Note that in case of vGPU, FLA VAS is allocated in guests, thus
129     // pKernelBus->flaInfo.bFlaAllocated shouldn't be set in the host. We allow
130     // FM running in the host to override the FLA range as we expect device
131     // to be idle when the vGPU partitions are being configured.
132     //
133     if (pKernelBus->flaInfo.bFlaAllocated)
134     {
135         // check if FM has previously allocated for the same FLA range
136         if (!kbusVerifyFlaRange_HAL(pGpu, pKernelBus, pParams->base, pParams->size))
137             return NV_ERR_IN_USE;
138 
139         return NV_OK;
140     }
141 
142     //
143     // We don't have to care about Arch Mods/pre-silicon verification scenarios
144     // for programming FLA base address to scratch register
145     //
146     NV_CHECK_OR_RETURN(LEVEL_SILENT, pKernelNvlink != NULL, NV_ERR_NOT_SUPPORTED);
147 
148     status  = knvlinkSetUniqueFlaBaseAddress(pGpu, pKernelNvlink, pParams->base);
149     if (status != NV_OK)
150     {
151         NV_PRINTF(LEVEL_ERROR,
152                   "Failed to set FLA range because of invalid config for gpu: %x\n",
153                   pGpu->gpuInstance);
154         return status;
155     }
156 
157     status = kbusAllocateFlaVaspace_HAL(pGpu, pKernelBus, pParams->base, pParams->size);
158     if (status != NV_OK)
159     {
160         NV_PRINTF(LEVEL_ERROR,
161                   "Allocating new FLA Vaspace failed, status: %x, gpu: %x \n",
162                   status, pGpu->gpuInstance);
163         return status;
164     }
165 
166     return NV_OK;
167 }
168 
169 NV_STATUS
170 subdeviceCtrlCmdFlaRange_IMPL
171 (
172     Subdevice *pSubdevice,
173     NV2080_CTRL_FLA_RANGE_PARAMS *pParams
174 )
175 {
176     NV_STATUS status = NV_OK;
177     OBJGPU   *pGpu = GPU_RES_GET_GPU(pSubdevice);
178     KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
179     KernelMIGManager *pKernelMIGManager;
180     NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice);
181     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
182 
183     NV_ASSERT_OR_RETURN(pCallContext != NULL, NV_ERR_INVALID_STATE);
184 
185     LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
186 
187     if (!rmclientIsCapableOrAdminByHandle(hClient,
188                                           NV_RM_CAP_EXT_FABRIC_MGMT,
189                                           pCallContext->secInfo.privLevel))
190     {
191         return NV_ERR_INSUFFICIENT_PERMISSIONS;
192     }
193 
194     if (!kbusIsFlaSupported(pKernelBus))
195     {
196         NV_PRINTF(LEVEL_INFO,
197                   "FLA is not supported in this platform\n");
198 
199         return NV_ERR_NOT_SUPPORTED;
200     }
201 
202     if (!ONEBITSET(pParams->mode))
203         return NV_ERR_INVALID_ARGUMENT;
204 
205     //
206     // Even if FLA is enabled, it is not compatible with MIG memory partitioning
207     // If MIG memory partitioning is enabled, we should not allow FLA address
208     // range creation
209     //
210     pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
211     if ((pKernelMIGManager != NULL) && !kmigmgrIsMIGNvlinkP2PSupported(pGpu, pKernelMIGManager))
212     {
213         NV_PRINTF(LEVEL_INFO,
214                   "ERROR: FLA cannot be enabled with peer support disabled with MIG\n");
215         return NV_ERR_NOT_SUPPORTED;
216     }
217 
218     switch (pParams->mode)
219     {
220         case NV2080_CTRL_FLA_RANGE_PARAMS_MODE_INITIALIZE:
221             status = _subdeviceFlaRangeModeInit(pSubdevice, pParams);
222             break;
223 
224         // Deprecated as no client should be able to invoke this control option.
225         case NV2080_CTRL_FLA_RANGE_PARAMS_MODE_DESTROY:
226             status = NV_OK;
227             break;
228 
229         case NV2080_CTRL_FLA_RANGE_PARAMS_MODE_HOST_MANAGED_VAS_INITIALIZE:
230             status = _subdeviceFlaRangeModeHostManagedVasInit(pSubdevice, pParams);
231             break;
232 
233         case NV2080_CTRL_FLA_RANGE_PARAMS_MODE_HOST_MANAGED_VAS_DESTROY:
234             status = _subdeviceFlaRangeModeHostManagedVasDestroy(pSubdevice, pParams);
235             break;
236 
237         default:
238             status = NV_ERR_INVALID_OPERATION;
239             break;
240     }
241 
242     return status;
243 }
244 
245 // Control call to manage FLA range in RM
246 NV_STATUS
247 subdeviceCtrlCmdFlaGetRange_IMPL
248 (
249     Subdevice *pSubdevice,
250     NV2080_CTRL_FLA_GET_RANGE_PARAMS *pParams
251 )
252 {
253     return NV_ERR_NOT_SUPPORTED;
254 }
255 
256 NV_STATUS
257 subdeviceCtrlCmdFlaGetFabricMemStats_IMPL
258 (
259     Subdevice *pSubdevice,
260     NV2080_CTRL_FLA_GET_FABRIC_MEM_STATS_PARAMS *pParams
261 )
262 {
263     OBJGPU         *pGpu       = GPU_RES_GET_GPU(pSubdevice);
264     FABRIC_VASPACE *pFabricVAS = NULL;
265     NV_STATUS       status = NV_OK;
266 
267     LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
268 
269     if (pGpu->pFabricVAS == NULL)
270     {
271         return NV_ERR_NOT_SUPPORTED;
272     }
273 
274     pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
275 
276     if (pFabricVAS->bRpcAlloc)
277     {
278         CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
279         RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams->pLegacyParams;
280 
281         NV_RM_RPC_CONTROL(pGpu,
282                           pRmCtrlParams->hClient,
283                           pRmCtrlParams->hObject,
284                           pRmCtrlParams->cmd,
285                           pRmCtrlParams->pParams,
286                           pRmCtrlParams->paramsSize,
287                           status);
288         return status;
289     }
290 
291     pParams->totalSize = fabricvaspaceGetUCFlaLimit(pFabricVAS) -
292                          fabricvaspaceGetUCFlaStart(pFabricVAS) + 1;
293 
294     return fabricvaspaceGetFreeHeap(pFabricVAS, &pParams->freeSize);
295 }
296 
297