1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "gpu/device/device.h"
25 #include "gpu/perf/kern_perf.h"
26 #include "core/locks.h"
27 #include "vgpu/rpc.h"
28 #include "nvRmReg.h"
29 #include "platform/sli/sli.h"
30
31 /* ------------------------ Macros ----------------------------------------- */
32 /* ------------------------ Public Class Interfaces ------------------------ */
33 /*!
34 * Initialize SW state corresponding to SLI GPU Boost synchronization.
35 *
36 * @param[in] pGpu GPU object pointer
37 * @param[in] pKernelPerf KernelPerf object pointer
38 *
39 * @return NV_OK
40 */
41 NV_STATUS
kperfGpuBoostSyncStateInit_IMPL(OBJGPU * pGpu,KernelPerf * pKernelPerf)42 kperfGpuBoostSyncStateInit_IMPL
43 (
44 OBJGPU *pGpu,
45 KernelPerf *pKernelPerf
46 )
47 {
48 NV_STATUS status = NV_OK;
49 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
50 NvU32 i;
51 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO_PARAMS ctrlParams = { 0 };
52
53 status = pRmApi->Control(pRmApi,
54 pGpu->hInternalClient,
55 pGpu->hInternalSubdevice,
56 NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_GET_INFO,
57 &ctrlParams,
58 sizeof(ctrlParams));
59
60 if (status != NV_OK)
61 {
62 NV_PRINTF(LEVEL_ERROR,
63 "Failed to read Sync Gpu Boost init state, status=0x%x\n",
64 status);
65 goto kperfGpuBoostSyncStateInit_IMPL_exit;
66 }
67
68 pKernelPerf->sliGpuBoostSync.hysteresisus = ctrlParams.hysteresisus;
69 pKernelPerf->sliGpuBoostSync.bHystersisEnable = ctrlParams.bHystersisEnable;
70 pKernelPerf->sliGpuBoostSync.bSliGpuBoostSyncEnable = ctrlParams.bSliGpuBoostSyncEnable;
71
72 // Initialize the GPU Boost synchronization limits.
73 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
74 {
75 pKernelPerf->sliGpuBoostSync.limits[i] = NV_U32_MAX;
76 }
77
78 kperfGpuBoostSyncStateInit_IMPL_exit:
79 return NV_OK;
80 }
81
82 /*!
83 * @copydoc kperfGpuBoostSyncActivate
84 */
85 NV_STATUS
kperfGpuBoostSyncActivate_IMPL(OBJGPU * pGpu,KernelPerf * pKernelPerf,NvBool bActivate)86 kperfGpuBoostSyncActivate_IMPL
87 (
88 OBJGPU *pGpu,
89 KernelPerf *pKernelPerf,
90 NvBool bActivate
91 )
92 {
93 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
94 NV_STATUS status = NV_OK;
95 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_CONTROL_PARAMS ctrlParams = { 0 };
96
97 ctrlParams.bActivate = bActivate;
98
99 status = pRmApi->Control(pRmApi,
100 pGpu->hInternalClient,
101 pGpu->hInternalSubdevice,
102 NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_SET_CONTROL,
103 &ctrlParams,
104 sizeof(ctrlParams));
105
106 return status;
107 }
108
109 /*!
110 * Processes all GPU Boost PERF_LIMITs and applies most restrictive of them for
111 * SLI GPU Boost synchronization.
112 *
113 * @param[in] pGpu OBJGPU pointer
114 * @param[in] pParams
115 *
116 * @return NV_OK
117 * Synchronized PERF_LIMITs successfully applied or removed.
118 * @return Other unexpected errors
119 * Unexpected errors propagated from other functions.
120 */
121 NV_STATUS
kperfDoSyncGpuBoostLimits_IMPL(OBJGPU * pGpu,KernelPerf * pKernelPerf,NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS * pParams)122 kperfDoSyncGpuBoostLimits_IMPL
123 (
124 OBJGPU *pGpu,
125 KernelPerf *pKernelPerf,
126 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS *pParams
127 )
128 {
129 OBJSYS *pSys = SYS_GET_INSTANCE();
130 OBJGPUMGR *pGpuMgr = SYS_GET_GPUMGR(pSys);
131 OBJGPUBOOSTMGR *pBoostMgr = SYS_GET_GPUBOOSTMGR(pSys);
132 NV_STATUS status = NV_OK;
133 NvU64 currns = 0;
134 NvU64 diffns = 0;
135 NvBool bUpdate = NV_FALSE;
136 NvBool bBridgeless = NV_FALSE;
137 NV2080_CTRL_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS_PARAMS perfGpuBoostSyncParamsSet = { 0 };
138 NvBool bBcState;
139 NvU32 grpId;
140 NvU32 i;
141
142 bBcState = gpumgrGetBcEnabledStatus(pGpu);
143
144 gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
145
146 gpuboostmgrGetBoostGrpIdFromGpu(pBoostMgr, pGpu, &grpId);
147
148 if (!gpuboostmgrIsBoostGrpActive(pBoostMgr, grpId))
149 {
150 return status;
151 }
152
153 NV_CHECK_OR_RETURN(LEVEL_ERROR, (pKernelPerf != NULL), NV_ERR_INVALID_POINTER);
154
155 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
156 {
157 pKernelPerf->sliGpuBoostSync.limits[i] = pParams->currLimits[i];
158 }
159 pKernelPerf->sliGpuBoostSync.bBridgeless = pParams->bBridgeless;
160
161 portMemSet(perfGpuBoostSyncParamsSet.currLimits, NV_U8_MAX, sizeof(perfGpuBoostSyncParamsSet.currLimits));
162
163 //
164 // NOTE:
165 // One will see a pattern of forks in this file:
166 // if (boost group active)
167 // Gpu Boost Loop
168 // else
169 // SLI Loop
170 // This is a temporary change to introduce Sync Gpu boost Manager to the SLI Boost framework.
171 // The goal eventually is to replace SLI GPU Boost with Sync GPU Boost.
172 // WDDM KMD and UMDs needs to change for that
173 //
174
175 if (gpuboostmgrIsBoostGrpActive(pBoostMgr, grpId))
176 {
177 OBJGPU *pGpuItr = NULL;
178
179 GPUBOOSTMGR_ITR_START(pBoostMgr, grpId, pGpuItr)
180 {
181 pKernelPerf = GPU_GET_KERNEL_PERF(pGpuItr);
182 NV_CHECK_OR_RETURN(LEVEL_ERROR, (pKernelPerf != NULL), NV_ERR_INVALID_POINTER);
183
184 // Find min of all GPU Boost PERF_LIMITs across all the GPUs.
185 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
186 {
187 perfGpuBoostSyncParamsSet.currLimits[i] = NV_MIN(perfGpuBoostSyncParamsSet.currLimits[i], pKernelPerf->sliGpuBoostSync.limits[i]);
188 }
189
190 if (pKernelPerf->sliGpuBoostSync.bBridgeless)
191 {
192 bBridgeless = NV_TRUE;
193 }
194
195 }
196 GPUBOOSTMGR_ITR_END
197 }
198 else
199 {
200 }
201
202 // Enable hysteresis algorithm, if required.
203 if ((pKernelPerf != NULL) &&
204 (pKernelPerf->sliGpuBoostSync.bHystersisEnable))
205 {
206 // Get current tick.
207 osGetPerformanceCounter(&currns);
208
209 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
210 {
211 // If GPU Boost PERF_LIMITs are being lowered, immediately synchronize.
212 if (perfGpuBoostSyncParamsSet.currLimits[i] < pGpuMgr->sliGpuBoostSync.prevLimits[i])
213 {
214 bUpdate = NV_TRUE;
215 }
216 else if (perfGpuBoostSyncParamsSet.currLimits[i] > pGpuMgr->sliGpuBoostSync.prevLimits[i])
217 {
218 // Otherwise, synchronize only if specified time has been elapsed.
219 diffns = currns - pGpuMgr->sliGpuBoostSync.prevChangeTsns;
220
221 if ((diffns / 1000) > pKernelPerf->sliGpuBoostSync.hysteresisus)
222 {
223 bUpdate = NV_TRUE;
224 }
225 }
226 }
227
228 // Update previous history and apply SLI GPU Boost PERF_LIMITs.
229 if (bUpdate)
230 {
231 pGpuMgr->sliGpuBoostSync.prevChangeTsns = currns;
232
233 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
234 {
235 pGpuMgr->sliGpuBoostSync.prevLimits[i] = perfGpuBoostSyncParamsSet.currLimits[i];
236 }
237 }
238 else
239 {
240 return status;
241 }
242 }
243
244 perfGpuBoostSyncParamsSet.flags = pParams->flags;
245 perfGpuBoostSyncParamsSet.bBridgeless = bBridgeless;
246
247 if (gpuboostmgrIsBoostGrpActive(pBoostMgr, grpId))
248 {
249 OBJGPU *pGpuItr = NULL;
250
251 GPUBOOSTMGR_ITR_START(pBoostMgr, grpId, pGpuItr)
252 {
253 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
254
255 status = pRmApi->Control(pRmApi,
256 pGpuItr->hInternalClient,
257 pGpuItr->hInternalSubdevice,
258 NV2080_CTRL_CMD_INTERNAL_PERF_GPU_BOOST_SYNC_SET_LIMITS,
259 &perfGpuBoostSyncParamsSet,
260 sizeof(perfGpuBoostSyncParamsSet));
261 }
262 GPUBOOSTMGR_ITR_END
263 }
264 else
265 {
266 }
267
268 gpumgrSetBcEnabledStatus(pGpu, bBcState);
269
270 return status;
271 }
272
273 /*!
274 * Update the bridgeless info.
275 *
276 * @param[in] pGpu GPU object pointer
277 * @param[in] bBridgeless Latest bridgeless info that should be set
278 *
279 * @return NV_OK
280 */
281 NV_STATUS
kPerfGpuBoostSyncBridgelessUpdateInfo(OBJGPU * pGpu,NvBool bBridgeless)282 kPerfGpuBoostSyncBridgelessUpdateInfo
283 (
284 OBJGPU *pGpu,
285 NvBool bBridgeless
286 )
287 {
288 KernelPerf *pKernelPerf = GPU_GET_KERNEL_PERF(pGpu);
289
290 if (pKernelPerf != NULL)
291 {
292 pKernelPerf->sliGpuBoostSync.bBridgeless = bBridgeless;
293 }
294
295 return NV_OK;
296 }
297
298 /*!
299 * Helper routine to toggle the state of Sync Gpu Boost Algorithm using SGBG infrastructure
300 * @param[in] bActivate NV_TRUE if we want to turn the algorithm on, NV_FALSE otherwise
301 */
302 NV_STATUS
kperfGpuBoostSyncStateUpdate(OBJGPUBOOSTMGR * pBoostMgr,NvU32 boostGroupId,NvBool bActivate)303 kperfGpuBoostSyncStateUpdate
304 (
305 OBJGPUBOOSTMGR *pBoostMgr,
306 NvU32 boostGroupId,
307 NvBool bActivate
308 )
309 {
310 OBJSYS *pSys = SYS_GET_INSTANCE();
311 KernelPerf *pKernelPerf = NULL;
312 NV_STATUS status = NV_OK;
313 NvU32 i;
314
315 // No need to activate again if refCount is greater than 1.
316 if (1 < pBoostMgr->pBoostGroups[boostGroupId].refCount)
317 {
318 NV_PRINTF(LEVEL_ERROR,
319 "Trying to activate and already active Sync GPU Boost Group = 0x%08x.\n",
320 boostGroupId);
321 DBG_BREAKPOINT();
322 return status;
323 }
324
325 //
326 // Trigger state change per GPU
327 // We need to acquire GPU locks here as this will tough GPU state.
328 //
329
330 // LOCK: acquire GPUs lock
331 if (NV_OK == rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_GPU))
332 {
333 if (NV_OK == osAcquireRmSema(pSys->pSema))
334 {
335 OBJGPU *pGpuItr = NULL;
336
337 GPUBOOSTMGR_ITR_START(pBoostMgr, boostGroupId, pGpuItr)
338 {
339 pKernelPerf = GPU_GET_KERNEL_PERF(pGpuItr);
340
341 if ((pKernelPerf != NULL) &&
342 (pKernelPerf->sliGpuBoostSync.bSliGpuBoostSyncEnable))
343 {
344 status = kperfGpuBoostSyncActivate(pGpuItr, pKernelPerf, bActivate);
345
346 if (status == NV_OK)
347 {
348 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
349 {
350 pKernelPerf->sliGpuBoostSync.limits[i] = NV_U32_MAX;
351 }
352 }
353 }
354 else
355 {
356 status = NV_ERR_INVALID_STATE;
357 }
358 if (NV_OK != status)
359 {
360 OBJGPU *pGpuItr2 = NULL;
361 NV_PRINTF(LEVEL_ERROR,
362 "Failed to toggle Sync Gpu Boost state on Gpu 0x%08x\n",
363 pGpuItr->gpuId);
364 DBG_BREAKPOINT();
365
366 // Toggle back the Sync Gpu Boost state of all the GPUs so far
367 GPUBOOSTMGR_ITR_START(pBoostMgr, boostGroupId, pGpuItr2)
368 {
369 pKernelPerf = GPU_GET_KERNEL_PERF(pGpuItr2);
370
371 if ((pKernelPerf != NULL) &&
372 (pKernelPerf->sliGpuBoostSync.bSliGpuBoostSyncEnable))
373 {
374 status = kperfGpuBoostSyncActivate(pGpuItr2, pKernelPerf, !bActivate);
375
376 if (status == NV_OK)
377 {
378 for (i = 0; i < NV2080_CTRL_INTERNAL_PERF_SYNC_GPU_BOOST_LIMITS_NUM; i++)
379 {
380 pKernelPerf->sliGpuBoostSync.limits[i] = NV_U32_MAX;
381 }
382 }
383 }
384 // Intentionaly ignoring the status as we want to rollback the algorithm
385 // activation and return previously failing status
386
387 if (pGpuItr == pGpuItr2)
388 {
389 // break from unwind/cleanup loop
390 break;
391 }
392 }
393 GPUBOOSTMGR_ITR_END
394
395 // break from outer iterator
396 break;
397 }
398 }
399 GPUBOOSTMGR_ITR_END
400 }
401 else
402 {
403 NV_PRINTF(LEVEL_ERROR, "OS Semaphore acquire failed\n");
404 status = NV_ERR_STATE_IN_USE;
405 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
406 goto kperfSliGpuBoostSyncStateUpdate_exit;
407 }
408
409 // UNLOCK: release GPUs lock
410 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
411 }
412 else
413 {
414 NV_PRINTF(LEVEL_ERROR, "GPU lock acquire failed\n");
415 status = NV_ERR_STATE_IN_USE;
416 goto kperfSliGpuBoostSyncStateUpdate_exit;
417 }
418
419 kperfSliGpuBoostSyncStateUpdate_exit:
420 return status;
421 }
422