1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2013 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <stddef.h>
25
26 #include "nvkms-dma.h"
27 #include "nvkms-utils.h"
28 #include "nvkms-rmapi.h"
29 #include "class/cl917d.h" // NV917DDispControlDma, NV917D_DMA_*
30 #include <ctrl/ctrl0080/ctrl0080dma.h> // NV0080_CTRL_CMD_DMA_FLUSH
31 #include "nvos.h"
32
33 #define NV_DMA_PUSHER_CHASE_PAD 5
34 #define NV_EVO_NOTIFIER_SHORT_TIMEOUT_USEC 3000000 // 3 seconds
35
36 static void EvoCoreKickoff(NVDmaBufferEvoPtr push_buffer, NvU32 putOffset);
37
nvDmaKickoffEvo(NVEvoChannelPtr pChannel)38 void nvDmaKickoffEvo(NVEvoChannelPtr pChannel)
39 {
40 NVDmaBufferEvoPtr p = &pChannel->pb;
41 NvU32 putOffset = (NvU32)((char *)p->buffer - (char *)p->base);
42
43 if (p->put_offset == putOffset) {
44 return;
45 }
46
47 EvoCoreKickoff(p, putOffset);
48 }
49
EvoCoreKickoff(NVDmaBufferEvoPtr push_buffer,NvU32 putOffset)50 static void EvoCoreKickoff(NVDmaBufferEvoPtr push_buffer, NvU32 putOffset)
51 {
52 NVEvoDmaPtr pDma = &push_buffer->dma;
53 int i;
54
55 nvAssert(putOffset % 4 == 0);
56 nvAssert(putOffset <= push_buffer->offset_max);
57
58 /* If needed, copy the chunk to be kicked off into each GPU's FB */
59 if (pDma->isBar1Mapping) {
60 NVDevEvoPtr pDevEvo = push_buffer->pDevEvo;
61 int sd;
62
63 NV0080_CTRL_DMA_FLUSH_PARAMS flushParams = { 0 };
64 NvU32 ret;
65
66 NvU32 *endAddress;
67
68 if (putOffset < push_buffer->put_offset) {
69 /* If we've wrapped, copy to the end of the pushbuffer */
70 nvAssert(putOffset == 0);
71 endAddress = push_buffer->base + push_buffer->offset_max /
72 sizeof(NvU32);
73 } else {
74 endAddress = push_buffer->buffer;
75 }
76
77 for (sd = 0; sd < pDevEvo->numSubDevices; sd++) {
78 NvU32 startOffset = push_buffer->put_offset / sizeof(NvU32);
79
80 NvU32 *src = push_buffer->base;
81 NvU32 *dst = pDma->subDeviceAddress[sd];
82
83 nvAssert(dst != NULL);
84
85 src += startOffset;
86 dst += startOffset;
87 while (src < endAddress) {
88 *dst++ = *src++;
89 }
90 }
91
92 /*
93 * Finally, tell RM to flush so that the data actually lands in FB
94 * before telling the GPU to fetch it.
95 */
96 flushParams.targetUnit = DRF_DEF(0080_CTRL_DMA, _FLUSH_TARGET,
97 _UNIT_FB, _ENABLE);
98
99 ret = nvRmApiControl(nvEvoGlobal.clientHandle,
100 pDevEvo->deviceHandle,
101 NV0080_CTRL_CMD_DMA_FLUSH,
102 &flushParams, sizeof(flushParams));
103 if (ret != NVOS_STATUS_SUCCESS) {
104 nvAssert(!"NV0080_CTRL_CMD_DMA_FLUSH failed");
105 }
106 }
107
108 #if NVCPU_IS_X86_64
109 __asm__ __volatile__ ("sfence\n\t" : : : "memory");
110 #elif NVCPU_IS_FAMILY_ARM
111 __asm__ __volatile__ ("dsb sy\n\t" : : : "memory");
112 #endif
113
114 /* Kick off all push buffers */
115 push_buffer->put_offset = putOffset;
116 for (i = 0; i < push_buffer->num_channels; i++) {
117 void *pControl = push_buffer->control[i];
118 nvDmaStorePioMethod(pControl, NV917D_PUT, putOffset);
119 }
120 }
121
122 /* Read GET from an EVO core channel */
EvoCoreReadGet(NVDmaBufferEvoPtr push_buffer,int sd)123 static NvU32 EvoCoreReadGet(NVDmaBufferEvoPtr push_buffer, int sd)
124 {
125 void *pControl = push_buffer->control[sd];
126 return nvDmaLoadPioMethod(pControl, NV917D_GET);
127 }
128
129 /* Read GET for all devices and return the minimum or maximum*/
EvoReadGetOffset(NVDmaBufferEvoPtr push_buffer,NvBool minimum)130 static NvU32 EvoReadGetOffset(NVDmaBufferEvoPtr push_buffer, NvBool minimum)
131 {
132 int i;
133 NvU32 get, bestGet = 0;
134 NvS32 distanceToPut, minmaxDistanceToPut = (minimum ?
135 0 :
136 (push_buffer->dma.limit + 1));
137
138 if (push_buffer->num_channels <= 1) {
139 return EvoCoreReadGet(push_buffer, 0);
140 }
141
142 for (i =0; i < push_buffer->num_channels; i++) {
143 get = EvoCoreReadGet(push_buffer, i);
144
145 /* Compute distance to put, accounting for wraps */
146 distanceToPut = push_buffer->put_offset - get;
147 if (distanceToPut < 0)
148 distanceToPut += push_buffer->dma.limit + 1;
149
150 /* Accumulate the maximum distance to put and the corresponding get. */
151 if ((minimum && (distanceToPut >= minmaxDistanceToPut)) ||
152 (!minimum && (distanceToPut <= minmaxDistanceToPut))) {
153 minmaxDistanceToPut = distanceToPut;
154 bestGet = get;
155 }
156 }
157 return bestGet;
158 }
159
nvEvoPollForEmptyChannel(NVEvoChannelPtr pChannel,NvU32 sd,NvU64 * pStartTime,const NvU32 timeout)160 NvBool nvEvoPollForEmptyChannel(NVEvoChannelPtr pChannel, NvU32 sd,
161 NvU64 *pStartTime, const NvU32 timeout)
162 {
163 NVDmaBufferEvoPtr push_buffer = &pChannel->pb;
164
165 do {
166 if (EvoCoreReadGet(push_buffer, sd) == push_buffer->put_offset) {
167 break;
168 }
169
170 if (nvExceedsTimeoutUSec(push_buffer->pDevEvo, pStartTime, timeout)) {
171 return FALSE;
172 }
173
174 nvkms_yield();
175 } while (TRUE);
176
177 return TRUE;
178 }
179
nvEvoMakeRoom(NVEvoChannelPtr pChannel,NvU32 count)180 void nvEvoMakeRoom(NVEvoChannelPtr pChannel, NvU32 count)
181 {
182 NVDmaBufferEvoPtr push_buffer = &pChannel->pb;
183 NvU32 getOffset;
184 NvU32 putOffset;
185 NvU64 startTime = 0;
186 const NvU64 timeout = 5000000; /* 5 seconds */
187
188 putOffset = (NvU32) ((char *)push_buffer->buffer -
189 (char *)push_buffer->base);
190
191 if (putOffset >= push_buffer->offset_max) {
192 *(push_buffer->buffer) = 0x20000000;
193 push_buffer->buffer = push_buffer->base;
194 nvDmaKickoffEvo(pChannel);
195 putOffset = 0;
196 }
197
198 while (1) {
199 getOffset = EvoReadGetOffset(push_buffer, TRUE);
200
201 if (putOffset >= getOffset) {
202 push_buffer->fifo_free_count =
203 (push_buffer->offset_max - putOffset) >> 2;
204
205 if (push_buffer->fifo_free_count <= count) {
206 if (getOffset) {
207 *(push_buffer->buffer) = 0x20000000;
208 push_buffer->buffer = push_buffer->base;
209 nvDmaKickoffEvo(pChannel);
210 putOffset = 0;
211 }
212 else if (putOffset != push_buffer->put_offset) {
213 nvDmaKickoffEvo(pChannel);
214 // Put offset will have changed if a tail was inserted.
215 putOffset = push_buffer->put_offset;
216 }
217 }
218 }
219 else {
220 getOffset = (getOffset > push_buffer->offset_max) ?
221 push_buffer->offset_max : getOffset;
222
223 if ((putOffset + (NV_DMA_PUSHER_CHASE_PAD * 4)) >= getOffset)
224 push_buffer->fifo_free_count = 0;
225 else
226 push_buffer->fifo_free_count =
227 ((getOffset - putOffset) >> 2) - 1;
228 }
229 if (push_buffer->fifo_free_count > count) {
230 break;
231 }
232
233 /*
234 * If we have been waiting too long, print an error message. There
235 * isn't much we can do as currently structured, so just reset
236 * startTime.
237 */
238 if (nvExceedsTimeoutUSec(push_buffer->pDevEvo, &startTime, timeout)) {
239 nvEvoLogDev(push_buffer->pDevEvo, EVO_LOG_ERROR,
240 "Error while waiting for GPU progress: "
241 "0x%08x:%d %d:%d:%d:%d",
242 pChannel->hwclass, pChannel->instance,
243 count, push_buffer->fifo_free_count, getOffset, putOffset);
244 startTime = 0;
245 }
246
247 nvkms_yield();
248 }
249 }
250
EvoWriteNotifier(volatile NvU32 * pNotifier,NvU32 value)251 static inline void EvoWriteNotifier(volatile NvU32 *pNotifier, NvU32 value)
252 {
253 /*
254 * Note that we don't need to flush to vidmem here; any subsequent GPU
255 * write will always be triggered by kicking off pushbuffer methods,
256 * which will perform a general FB flush. This does assume that the
257 * pushbuffer and its associated notifier surfaces are either both in
258 * sysmem or both in vidmem, however.
259 */
260
261 *pNotifier = value;
262 }
263
264 /* Write the EVO core notifier at the given offset to the given value. */
nvWriteEvoCoreNotifier(const NVDispEvoRec * pDispEvo,NvU32 offset,NvU32 value)265 void nvWriteEvoCoreNotifier(
266 const NVDispEvoRec *pDispEvo,
267 NvU32 offset,
268 NvU32 value)
269 {
270 NVDevEvoPtr pDevEvo = pDispEvo->pDevEvo;
271 const NvU32 sd = pDispEvo->displayOwner;
272 NVEvoDmaPtr pSubChannel = &pDevEvo->core->notifiersDma[sd];
273 volatile NvU32 *pNotifiers = pSubChannel->subDeviceAddress[sd];
274
275 EvoWriteNotifier(pNotifiers + offset, value);
276 }
277
EvoCheckNotifier(const NVDispEvoRec * pDispEvo,NvU32 offset,NvU32 done_base_bit,NvU32 done_extent_bit,NvU32 done_value,NvBool wait)278 static NvBool EvoCheckNotifier(const NVDispEvoRec *pDispEvo,
279 NvU32 offset, NvU32 done_base_bit,
280 NvU32 done_extent_bit, NvU32 done_value,
281 NvBool wait)
282 {
283 const NvU32 sd = pDispEvo->displayOwner;
284 NVDevEvoPtr pDevEvo = pDispEvo->pDevEvo;
285 NVEvoDmaPtr pSubChannel = &pDevEvo->core->notifiersDma[sd];
286 NVDmaBufferEvoPtr p = &pDevEvo->core->pb;
287 volatile NvU32 *pNotifier;
288 NvU64 startTime = 0;
289
290 pNotifier = pSubChannel->subDeviceAddress[sd];
291
292 nvAssert(pNotifier != NULL);
293 pNotifier += offset;
294
295 // While the completion notifier is not set to done_true
296 do {
297 const NvU32 val = *pNotifier;
298 const NvU32 done_mask = DRF_SHIFTMASK(done_extent_bit:done_base_bit);
299 const NvU32 done_val = done_value << done_base_bit;
300
301 if ((val & done_mask) == done_val) {
302 return TRUE;
303 }
304
305 if (!wait) {
306 return FALSE;
307 }
308
309 if (nvExceedsTimeoutUSec(
310 pDevEvo,
311 &startTime,
312 NV_EVO_NOTIFIER_SHORT_TIMEOUT_USEC) &&
313 (p->put_offset == EvoCoreReadGet(p, sd)))
314 {
315 nvEvoLogDisp(pDispEvo, EVO_LOG_WARN,
316 "Lost display notification (%d:0x%08x); "
317 "continuing.", sd, val);
318 EvoWriteNotifier(pNotifier, done_value << done_base_bit);
319 return TRUE;
320 }
321
322 nvkms_yield();
323 } while (TRUE);
324 }
325
326 /*
327 * Used by NV_EVO_WAIT_FOR_NOTIFIER() and NV_EVO_WAIT_FOR_CAPS_NOTIFIER()
328 */
nvEvoWaitForCoreNotifier(const NVDispEvoRec * pDispEvo,NvU32 offset,NvU32 done_base_bit,NvU32 done_extent_bit,NvU32 done_value)329 void nvEvoWaitForCoreNotifier(const NVDispEvoRec *pDispEvo, NvU32 offset,
330 NvU32 done_base_bit, NvU32 done_extent_bit,
331 NvU32 done_value)
332 {
333 EvoCheckNotifier(pDispEvo, offset,
334 done_base_bit, done_extent_bit, done_value, TRUE);
335 }
336
337 /*
338 * Used by the EVO HAL IsNotifierComplete functions. Returns TRUE if the
339 * notifier is complete.
340 */
nvEvoIsCoreNotifierComplete(NVDispEvoPtr pDispEvo,NvU32 offset,NvU32 done_base_bit,NvU32 done_extent_bit,NvU32 done_value)341 NvBool nvEvoIsCoreNotifierComplete(NVDispEvoPtr pDispEvo, NvU32 offset,
342 NvU32 done_base_bit, NvU32 done_extent_bit,
343 NvU32 done_value)
344 {
345 return EvoCheckNotifier(pDispEvo,
346 offset, done_base_bit, done_extent_bit,
347 done_value, FALSE);
348 }
349
nvEvoSetSubdeviceMask(NVEvoChannelPtr pChannel,NvU32 mask)350 void nvEvoSetSubdeviceMask(NVEvoChannelPtr pChannel, NvU32 mask)
351 {
352 NVDmaBufferEvoPtr p = &pChannel->pb;
353
354 nvAssert(!nvDmaSubDevMaskMatchesCurrent(pChannel, mask));
355
356 p->currentSubDevMask = mask;
357
358 ASSERT_DRF_NUM(917D, _DMA, _SET_SUBDEVICE_MASK_VALUE, mask);
359
360 if (p->fifo_free_count <= 1) {
361 nvEvoMakeRoom(pChannel, 1);
362 }
363
364 nvDmaSetEvoMethodData(pChannel,
365 DRF_DEF(917D, _DMA, _OPCODE, _SET_SUBDEVICE_MASK) |
366 DRF_NUM(917D, _DMA, _SET_SUBDEVICE_MASK_VALUE, mask));
367 p->fifo_free_count -= 1;
368 }
369
370 /*!
371 * Reads CRC values from the notifier.
372 *
373 * This function will attempt to read in the first 'entry_count' CRC notifier
374 * entries that HW generated. The actual number of entries that are read may
375 * be less.
376 *
377 * \param[in] pCRC32Notifier Pointer to the CRC notifier memory.
378 * \param[in] entry_stride Stride of a single CRC notifier entry
379 * \param[in] entry_count Expected count of notifier entries to read
380 * \param[in] status_offset Offset for Status flags header in CRC notifier
381 * \param[in] field_count Number of fields to read from each CRC notifier
382 * entry.
383 * \param[in] flag_count Number of flags to read from the Status Header
384 * \param[in out] field_info Specifies the offset/base/extent info for each field.
385 * Each 'field_info' contains an output array for
386 * storing 'entry_count' field values.
387 * \param[in] flag_info Specifies the base/extent info for each flag.
388 * Each 'flag_info' contains a 'flag_type' for
389 * addressing error cases related to the flags.
390 *
391 * \return Returns the MIN(count, entry_count) of successfully
392 * read entries.
393 */
nvEvoReadCRC32Notifier(volatile NvU32 * pCRC32Notifier,NvU32 entry_stride,NvU32 entry_count,NvU32 status_offset,NvU32 field_count,NvU32 flag_count,const CRC32NotifierEntryRec * field_info,const CRC32NotifierEntryFlags * flag_info)394 NvU32 nvEvoReadCRC32Notifier(volatile NvU32 *pCRC32Notifier,
395 NvU32 entry_stride,
396 NvU32 entry_count,
397 NvU32 status_offset,
398 NvU32 field_count,
399 NvU32 flag_count,
400 const CRC32NotifierEntryRec *field_info,
401 const CRC32NotifierEntryFlags *flag_info)
402 {
403 NvU32 count = 0;
404 NvU32 i, j, k;
405
406 nvAssert(pCRC32Notifier != NULL);
407
408 // Iterate over flags (unique at start of the CRC32Notifier Struct)
409 for (k = 0; k < flag_count; k++) {
410 CRC32NotifierEntryFlags info = flag_info[k];
411 volatile NvU32 *pFlag = pCRC32Notifier + status_offset;
412 NvU32 flag_mask =
413 DRF_SHIFTMASK((info.flag_extent_bit):(info.flag_base_bit));
414 NvU32 flag = (*pFlag & flag_mask) >> info.flag_base_bit;
415
416 switch (info.flag_type)
417 {
418 case NVEvoCrc32NotifierFlagCount:
419 count = flag;
420 // entry_count is max of each field_frame_values[i] array
421 if (count > entry_count) {
422 nvEvoLog(EVO_LOG_WARN, "Too many CRC32 generated entries "
423 "(%d expected; %d found)", entry_count, count);
424 count = entry_count;
425 }
426 break;
427
428 case NVEvoCrc32NotifierFlagCrcOverflow:
429 if (flag) {
430 count = 0;
431 nvEvoLog(EVO_LOG_ERROR, "CRC Overflow occured, "
432 "CRC value unable to be processed fast enough.\n"
433 "Failing flag index in status_info array: %d",
434 k);
435
436 return count;
437 }
438 break;
439 }
440 }
441
442 // Iterate over each collection of fields, for count pairs of values
443 for (i = 0; i < count; i++) {
444 for (j = 0; j < field_count; j++) {
445 CRC32NotifierEntryRec info = field_info[j];
446 volatile NvU32 *pEntry = pCRC32Notifier + info.field_offset;
447 NvU32 field_mask =
448 DRF_SHIFTMASK((info.field_extent_bit):(info.field_base_bit));
449
450 info.field_frame_values[i].value =
451 (*pEntry & field_mask) >> info.field_base_bit;
452 info.field_frame_values[i].supported = TRUE;
453 }
454 pCRC32Notifier += entry_stride;
455 }
456
457 return count;
458 }
459
nvEvoResetCRC32Notifier(volatile NvU32 * pCRC32Notifier,NvU32 offset,NvU32 reset_base_bit,NvU32 reset_value)460 void nvEvoResetCRC32Notifier(volatile NvU32 *pCRC32Notifier,
461 NvU32 offset,
462 NvU32 reset_base_bit,
463 NvU32 reset_value)
464 {
465 const NvU32 reset_val = reset_value << reset_base_bit;
466
467 nvAssert(pCRC32Notifier != NULL);
468 pCRC32Notifier += offset;
469
470 EvoWriteNotifier(pCRC32Notifier, reset_val);
471 }
472
nvEvoWaitForCRC32Notifier(const NVDevEvoPtr pDevEvo,volatile NvU32 * pCRC32Notifier,NvU32 offset,NvU32 done_base_bit,NvU32 done_extent_bit,NvU32 done_value)473 NvBool nvEvoWaitForCRC32Notifier(const NVDevEvoPtr pDevEvo,
474 volatile NvU32 *pCRC32Notifier,
475 NvU32 offset,
476 NvU32 done_base_bit,
477 NvU32 done_extent_bit,
478 NvU32 done_value)
479 {
480 const NvU32 done_mask = DRF_SHIFTMASK(done_extent_bit:done_base_bit);
481 const NvU32 done_val = done_value << done_base_bit;
482 NvU64 startTime = 0;
483
484 nvAssert(pCRC32Notifier != NULL);
485 pCRC32Notifier += offset;
486
487 do {
488 const NvU32 status = *pCRC32Notifier;
489
490 if ((status & done_mask) == done_val) {
491 return TRUE;
492 }
493
494 if (nvExceedsTimeoutUSec(
495 pDevEvo,
496 &startTime,
497 NV_EVO_NOTIFIER_SHORT_TIMEOUT_USEC)) {
498 return FALSE;
499 }
500
501 nvkms_yield();
502
503 } while (TRUE);
504
505 return FALSE;
506 }
507