1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2004-2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /*!
25 * @file
26 * @brief This module contains the gpu control interfaces for the
27 * subdevice (NV20_SUBDEVICE_0) class. Subdevice-level control calls
28 * are directed unicast to the associated GPU.
29 * File contains ctrls related to TMR engine object
30 */
31
32 #include "core/core.h"
33
34
35 #include "core/locks.h"
36 #include "gpu/subdevice/subdevice.h"
37 #include "objtmr.h"
38 #include "rmapi/client.h"
39
40 #include "kernel/gpu/intr/intr.h"
41
42 //
43 // subdeviceCtrlCmdTimerCancel
44 //
45 // Lock Requirements:
46 // Assert that API lock and GPUs lock held on entry
47 //
48 NV_STATUS
subdeviceCtrlCmdTimerCancel_IMPL(Subdevice * pSubdevice)49 subdeviceCtrlCmdTimerCancel_IMPL
50 (
51 Subdevice *pSubdevice
52 )
53 {
54 OBJGPU *pGpu;
55 OBJTMR *pTmr;
56
57 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
58
59 if (pSubdevice == NULL)
60 {
61 return NV_ERR_INVALID_ARGUMENT;
62 }
63
64 pGpu = GPU_RES_GET_GPU(pSubdevice);
65 pTmr = GPU_GET_TIMER(pGpu);
66
67 if (pSubdevice->notifyActions[NV2080_NOTIFIERS_TIMER] != NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE)
68 {
69 tmrCancelCallback(pTmr, pSubdevice);
70 pSubdevice->notifyActions[NV2080_NOTIFIERS_TIMER] = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE;
71 }
72 return NV_OK;
73 }
74
75 static NV_STATUS
gpuControlTimerCallback(OBJGPU * pGpu,OBJTMR * pTmr,void * pData)76 gpuControlTimerCallback(OBJGPU *pGpu, OBJTMR *pTmr, void * pData)
77 {
78 Subdevice *pSubDevice = (Subdevice *) pData;
79 PEVENTNOTIFICATION pNotifyEvent = inotifyGetNotificationList(staticCast(pSubDevice, INotifier));
80
81 if (pSubDevice->notifyActions[NV2080_NOTIFIERS_TIMER] == NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE)
82 {
83 NV_PRINTF(LEVEL_INFO,
84 "callback is called but the timer is not scheduled\n");
85 return NV_ERR_INVALID_STATE;
86 }
87
88 // Mark the timer as processed (no self-rescheduling for now)
89 pSubDevice->notifyActions[NV2080_NOTIFIERS_TIMER] = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE;
90
91 // Find timer event
92 while ((pNotifyEvent != NULL) && (pNotifyEvent->NotifyIndex != NV2080_NOTIFIERS_TIMER))
93 {
94 pNotifyEvent = pNotifyEvent->Next;
95 }
96 if (pNotifyEvent == NULL)
97 {
98 NV_PRINTF(LEVEL_INFO, "timer event is missing\n");
99 return NV_ERR_INVALID_STATE;
100 }
101
102 // perform a direct callback to the client
103 if (pNotifyEvent->Data != NvP64_NULL)
104 {
105 NvU64 currentTime = tmrGetTime_HAL(pGpu, pTmr);
106 osEventNotification(pGpu, pNotifyEvent, NV2080_NOTIFIERS_TIMER,
107 ¤tTime, sizeof(currentTime));
108 }
109 else
110 {
111 NV_PRINTF(LEVEL_INFO, "timer callback pointer is missing\n");
112 return NV_ERR_INVALID_STATE;
113 }
114 return NV_OK;
115 }
116
117 static NV_STATUS
timerSchedule(Subdevice * pSubdevice,NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS * pTimerScheduleParams)118 timerSchedule
119 (
120 Subdevice *pSubdevice,
121 NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS *pTimerScheduleParams
122 )
123 {
124 OBJGPU *pGpu;
125 OBJTMR *pTmr;
126 PEVENTNOTIFICATION pNotifyEvent;
127
128 if (pSubdevice == NULL)
129 {
130 return NV_ERR_INVALID_ARGUMENT;
131 }
132
133 pGpu = GPU_RES_GET_GPU(pSubdevice);
134 pTmr = GPU_GET_TIMER(pGpu);
135
136 pNotifyEvent = inotifyGetNotificationList(staticCast(pSubdevice, INotifier));
137
138 if (pSubdevice->notifyActions[NV2080_NOTIFIERS_TIMER] != NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_DISABLE)
139 {
140 NV_PRINTF(LEVEL_INFO,
141 "gpuControlTimerCallback: the timer is already scheduled for this subdevice\n");
142 return NV_ERR_INVALID_STATE;
143 }
144
145 // Validate the timer event
146 while ((pNotifyEvent != NULL) && (pNotifyEvent->NotifyIndex != NV2080_NOTIFIERS_TIMER))
147 {
148 pNotifyEvent = pNotifyEvent->Next;
149 }
150 if (pNotifyEvent == NULL)
151 {
152 NV_PRINTF(LEVEL_INFO,
153 "gpuControlTimerCallback: timer event is missing\n");
154 return NV_ERR_INVALID_STATE;
155 }
156 if (((pNotifyEvent->NotifyType != NV01_EVENT_KERNEL_CALLBACK) && (pNotifyEvent->NotifyType != NV01_EVENT_KERNEL_CALLBACK_EX)) ||
157 (pNotifyEvent->Data == NvP64_NULL))
158 {
159 NV_PRINTF(LEVEL_INFO,
160 "gpuControlTimer: cmd 0x%x: callback function is missing\n",
161 NV2080_CTRL_CMD_TIMER_SCHEDULE);
162 return NV_ERR_INVALID_STATE;
163
164 }
165
166 // Mark the timer as processed (no self-rescheduling for now). Set the flag before calling the timer
167 // since callback may be called right away.
168 pSubdevice->notifyActions[NV2080_NOTIFIERS_TIMER] = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_SINGLE;
169
170 // schedule the timer
171 if (DRF_VAL(2080, _CTRL_TIMER_SCHEDULE_FLAGS, _TIME, pTimerScheduleParams->flags) == NV2080_CTRL_TIMER_SCHEDULE_FLAGS_TIME_ABS)
172 {
173 tmrScheduleCallbackAbs(pTmr, gpuControlTimerCallback, pSubdevice, pTimerScheduleParams->time_nsec, 0, 0);
174 }
175 else
176 {
177 tmrScheduleCallbackRel(pTmr, gpuControlTimerCallback, pSubdevice, pTimerScheduleParams->time_nsec, 0, 0);
178 }
179
180 return NV_OK;
181 }
182
183 //
184 // subdeviceCtrlCmdTimerSchedule
185 //
186 // Lock Requirements:
187 // Assert that API lock and GPUs lock held on entry
188 //
189 NV_STATUS
subdeviceCtrlCmdTimerSchedule_IMPL(Subdevice * pSubdevice,NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS * pParams)190 subdeviceCtrlCmdTimerSchedule_IMPL
191 (
192 Subdevice *pSubdevice,
193 NV2080_CTRL_CMD_TIMER_SCHEDULE_PARAMS *pParams
194 )
195 {
196 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
197 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
198
199 if (pRmCtrlParams->flags & NVOS54_FLAGS_IRQL_RAISED)
200 {
201 LOCK_ASSERT_AND_RETURN(rmDeviceGpuLockIsOwner(GPU_RES_GET_GPU(pSubdevice)->gpuInstance));
202 }
203 else
204 {
205 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
206 }
207
208 return timerSchedule(pSubdevice, pParams);
209 }
210
211 //
212 // subdeviceCtrlCmdTimerGetTime
213 //
214 // Lock Requirements:
215 // Assert that API lock and GPUs lock held on entry
216 // Timer callback list accessed in tmrService at DPC
217 //
218 NV_STATUS
subdeviceCtrlCmdTimerGetTime_IMPL(Subdevice * pSubdevice,NV2080_CTRL_TIMER_GET_TIME_PARAMS * pParams)219 subdeviceCtrlCmdTimerGetTime_IMPL
220 (
221 Subdevice *pSubdevice,
222 NV2080_CTRL_TIMER_GET_TIME_PARAMS *pParams
223 )
224 {
225 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
226 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
227
228 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
229 OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
230
231 if ((pRmCtrlParams->flags & NVOS54_FLAGS_IRQL_RAISED) &&
232 (pRmCtrlParams->flags & NVOS54_FLAGS_LOCK_BYPASS))
233 {
234 if (pTmr->tmrChangePending)
235 {
236 return NV_ERR_STATE_IN_USE;
237 }
238 }
239 else
240 {
241 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance));
242 }
243
244 tmrGetCurrentTime(pTmr, &pParams->time_nsec);
245
246 return NV_OK;
247 }
248
249 //
250 // subdeviceCtrlCmdTimerGetRegisterOffset
251 //
252 // Lock Requirements:
253 // Assert that API lock held on entry
254 // No GPUs lock
255 //
256 NV_STATUS
subdeviceCtrlCmdTimerGetRegisterOffset_IMPL(Subdevice * pSubdevice,NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS * pTimerRegOffsetParams)257 subdeviceCtrlCmdTimerGetRegisterOffset_IMPL
258 (
259 Subdevice *pSubdevice,
260 NV2080_CTRL_TIMER_GET_REGISTER_OFFSET_PARAMS *pTimerRegOffsetParams
261 )
262 {
263 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
264
265 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner());
266
267 return gpuGetRegBaseOffset_HAL(pGpu, NV_REG_BASE_TIMER, &pTimerRegOffsetParams->tmr_offset);
268 }
269
270 /*!
271 * @brief Provides correlation information between GPU time and CPU time.
272 *
273 * @param[in] pSubDevice
274 * @param[in] pParams
275 *
276 * @return NV_OK Success
277 * @return NV_ERR_INVALID_ARGUMENT Invalid argument
278 * @return NV_ERR_NOT_SUPPORTED Unsupported CPU clock id
279 */
280 NV_STATUS
subdeviceCtrlCmdTimerGetGpuCpuTimeCorrelationInfo_IMPL(Subdevice * pSubdevice,NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS * pParams)281 subdeviceCtrlCmdTimerGetGpuCpuTimeCorrelationInfo_IMPL
282 (
283 Subdevice *pSubdevice,
284 NV2080_CTRL_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO_PARAMS *pParams
285 )
286 {
287 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
288 OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
289 NV_STATUS status = NV_OK;
290 NvU8 i;
291 NvU32 sec, usec;
292
293 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
294
295 NV_CHECK_OR_RETURN(LEVEL_SILENT,
296 (pParams->sampleCount <= NV2080_CTRL_TIMER_GPU_CPU_TIME_MAX_SAMPLES),
297 NV_ERR_INVALID_ARGUMENT);
298
299 if (RMCFG_FEATURE_PLATFORM_GSP)
300 {
301 NV_ASSERT_OR_RETURN(
302 FLD_TEST_DRF(2080, _TIMER_GPU_CPU_TIME_CPU_CLK_ID, _PROCESSOR, _GSP,
303 pParams->cpuClkId),
304 NV_ERR_INVALID_ARGUMENT);
305 }
306 else if (FLD_TEST_DRF(2080, _TIMER_GPU_CPU_TIME_CPU_CLK_ID, _PROCESSOR, _GSP,
307 pParams->cpuClkId))
308 {
309 //
310 // If GSP time is requested, forward the whole request to GSP.
311 // This can only be supported in GSP-RM offload mode.
312 //
313 if (!IS_GSP_CLIENT(pGpu))
314 return NV_ERR_NOT_SUPPORTED;
315
316 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
317
318 return pRmApi->Control(pRmApi,
319 pGpu->hInternalClient,
320 pGpu->hInternalSubdevice,
321 NV2080_CTRL_CMD_TIMER_GET_GPU_CPU_TIME_CORRELATION_INFO,
322 pParams, sizeof(*pParams));
323 }
324 else
325 {
326 NV_CHECK_OR_RETURN(LEVEL_SILENT,
327 FLD_TEST_DRF(2080, _TIMER_GPU_CPU_TIME_CPU_CLK_ID, _PROCESSOR, _CPU,
328 pParams->cpuClkId),
329 NV_ERR_INVALID_ARGUMENT);
330 }
331
332 switch (DRF_VAL(2080, _TIMER_GPU_CPU_TIME_CPU_CLK_ID, _SOURCE, pParams->cpuClkId))
333 {
334 case NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_OSTIME:
335 {
336 for (i = 0; i < pParams->sampleCount; i++)
337 {
338 osGetCurrentTime(&sec, &usec);
339 pParams->samples[i].cpuTime = (((NvU64)sec) * 1000000) + usec;
340 status = tmrGetCurrentTime(pTmr,
341 &pParams->samples[i].gpuTime);
342 if (status != NV_OK)
343 {
344 NV_PRINTF(LEVEL_ERROR,
345 "Could not get GPU time. status=0x%08x\n",
346 status);
347 break;
348 }
349 }
350 break;
351 }
352
353 case NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_PLATFORM_API:
354 {
355 //
356 // As reading CPU time and GPU time is a serial process we need to
357 // have a technique to mitigate the effects of preemption so we read
358 // the timestamps in a zipper pattern like c G c G c G c into an
359 // array storing all 7 values, find the two c values closest together,
360 // and report the sync point as the average of those two c values and
361 // the G between them. One complication is that reading a GPU's PTIMER
362 // directly from the CPU must be done via two non-adjacent BAR0-mapped
363 // memory locations for the low 32 bits and high 32 bits, and there's
364 // no way to atomically get both. One way to fix this is to make the
365 // read of the GPU time do the high bits, the low bits, and the high
366 // bits again, and if the two high values differ, we repeat the process
367 // until Ghi1 and Ghi2 match Once Ghi1 and 2 match, we use that as
368 // the high bits and the lo bits & CPU time from the zipper.
369 //
370 const NvU32 numTimerSamples = 3; // We take (hardcoded) 3 gpu timestamps.
371 NvU32 gpuTimeLo[3]; // Array to hold num_timer_samples gpu timestamps.
372 NvU64 cpuTime[4]; // Array to hold num_timer_samples+1 cpu timestamps.
373 NvU64 min;
374 NvU32 closestPairBeginIndex;
375 NvU32 gpuTimeHiOld;
376 NvU32 gpuTimeHiNew;
377 NvU32 i;
378
379 gpuTimeHiNew = tmrReadTimeHiReg_HAL(pGpu, pTmr, NULL);
380
381 do
382 {
383 gpuTimeHiOld = gpuTimeHiNew;
384 for (i = 0; i < numTimerSamples; i++)
385 {
386
387 osGetPerformanceCounter(&cpuTime[i]);
388
389 gpuTimeLo[i] = tmrReadTimeLoReg_HAL(pGpu, pTmr, NULL);
390 }
391
392 osGetPerformanceCounter(&cpuTime[i]);
393
394 // Read GPU TIME_1(High) again to detect wrap around.
395 gpuTimeHiNew = tmrReadTimeHiReg_HAL(pGpu, pTmr, NULL);
396 } while (gpuTimeHiNew != gpuTimeHiOld);
397
398 // find i such that cpuTime[i+1] - cpuTime[i] is minimum
399 // i.e. find closest pair of cpuTime.
400 min = cpuTime[1] - cpuTime[0];
401 closestPairBeginIndex = 0;
402 for (i = 0; i < numTimerSamples; i++)
403 {
404 if ((cpuTime[i+1] - cpuTime[i]) < min)
405 {
406 closestPairBeginIndex = i;
407 min = cpuTime[i+1] - cpuTime[i];
408 }
409 }
410
411 pParams->samples[0].gpuTime = ((((NvU64)gpuTimeHiNew) << 32) |
412 gpuTimeLo[closestPairBeginIndex]);
413 pParams->samples[0].cpuTime = (cpuTime[closestPairBeginIndex] +
414 cpuTime[closestPairBeginIndex + 1])/2;
415 NV_PRINTF(LEVEL_INFO,
416 "GPUTime = %llx CPUTime = %llx\n",
417 pParams->samples[0].gpuTime, pParams->samples[0].cpuTime);
418 break;
419 }
420
421 case NV2080_TIMER_GPU_CPU_TIME_CPU_CLK_ID_TSC:
422 {
423 for (i = 0; i < pParams->sampleCount; i++)
424 {
425 status = tmrGetGpuAndCpuTimestampPair_HAL(pGpu, pTmr, &pParams->samples[i].gpuTime, &pParams->samples[i].cpuTime);
426 if (status != NV_OK)
427 {
428 NV_PRINTF(LEVEL_ERROR,
429 "Could not get CPU GPU time. status=0x%08x\n",
430 status);
431 break;
432 }
433 }
434 break;
435 }
436 default:
437 {
438 status = NV_ERR_NOT_SUPPORTED;
439 break;
440 }
441 }
442
443 return status;
444 }
445
446 /*!
447 * @brief Set the frequency to update GR time stamp to default or max.
448 *
449 * The GR tick frequency will be restored to default
450 * only when no client has a pending request to increase.
451 *
452 * Lock Requirements:
453 * Assert that API lock held on entry
454 * No GPUs lock
455 *
456 * @param[in] pSubDevice
457 * @param[in] pParams
458 *
459 * @return NV_OK Success
460 * NV_ERR_INVALID_ARGUMENT Invalid Argument
461 * NV_ERR_NOT_SUPPORTED Not Supported
462 * Other errors from refcntRequestReference() or refcntReleaseReferences()
463 */
464 NV_STATUS
subdeviceCtrlCmdTimerSetGrTickFreq_IMPL(Subdevice * pSubdevice,NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS * pParams)465 subdeviceCtrlCmdTimerSetGrTickFreq_IMPL
466 (
467 Subdevice *pSubdevice,
468 NV2080_CTRL_CMD_TIMER_SET_GR_TICK_FREQ_PARAMS *pParams
469 )
470 {
471 OBJGPU *pGpu = GPU_RES_GET_GPU(pSubdevice);
472 OBJTMR *pTmr = GPU_GET_TIMER(pGpu);
473 NvHandle hClient = RES_GET_CLIENT_HANDLE(pSubdevice);
474 NV_STATUS status;
475 OBJREFCNT *pRefcnt;
476 NvHandle hSubDevice;
477
478 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner());
479
480 if (pSubdevice == NULL || pTmr == NULL)
481 {
482 return NV_ERR_INVALID_ARGUMENT;
483 }
484
485 pRefcnt = pTmr->pGrTickFreqRefcnt;
486
487 hSubDevice = RES_GET_HANDLE(pSubdevice);
488 if (pParams->bSetMaxFreq)
489 {
490 status = refcntRequestReference(pRefcnt,
491 NV_REQUESTER_CLIENT_OBJECT(hClient, hSubDevice),
492 REFCNT_STATE_ENABLED, NV_FALSE);
493 pSubdevice->bMaxGrTickFreqRequested = NV_TRUE;
494 }
495 else
496 {
497 status = refcntReleaseReferences(pRefcnt,
498 NV_REQUESTER_CLIENT_OBJECT(hClient, hSubDevice),
499 NV_TRUE);
500 pSubdevice->bMaxGrTickFreqRequested = NV_FALSE;
501 }
502 return status;
503 }
504
505