xref: /dragonfly/sys/dev/drm/drm_vblank.c (revision 3f2dd94a)
1 /*
2  * drm_irq.c IRQ and vblank support
3  *
4  * \author Rickard E. (Rik) Faith <faith@valinux.com>
5  * \author Gareth Hughes <gareth@valinux.com>
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the "Software"),
9  * to deal in the Software without restriction, including without limitation
10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11  * and/or sell copies of the Software, and to permit persons to whom the
12  * Software is furnished to do so, subject to the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the next
15  * paragraph) shall be included in all copies or substantial portions of the
16  * Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
21  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24  * OTHER DEALINGS IN THE SOFTWARE.
25  */
26 
27 #include <drm/drm_vblank.h>
28 #include <drm/drmP.h>
29 #include <linux/export.h>
30 
31 #include "drm_trace.h"
32 #include "drm_internal.h"
33 
34 /**
35  * DOC: vblank handling
36  *
37  * Vertical blanking plays a major role in graphics rendering. To achieve
38  * tear-free display, users must synchronize page flips and/or rendering to
39  * vertical blanking. The DRM API offers ioctls to perform page flips
40  * synchronized to vertical blanking and wait for vertical blanking.
41  *
42  * The DRM core handles most of the vertical blanking management logic, which
43  * involves filtering out spurious interrupts, keeping race-free blanking
44  * counters, coping with counter wrap-around and resets and keeping use counts.
45  * It relies on the driver to generate vertical blanking interrupts and
46  * optionally provide a hardware vertical blanking counter.
47  *
48  * Drivers must initialize the vertical blanking handling core with a call to
49  * drm_vblank_init(). Minimally, a driver needs to implement
50  * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
51  * drm_crtc_handle_vblank() in it's vblank interrupt handler for working vblank
52  * support.
53  *
54  * Vertical blanking interrupts can be enabled by the DRM core or by drivers
55  * themselves (for instance to handle page flipping operations).  The DRM core
56  * maintains a vertical blanking use count to ensure that the interrupts are not
57  * disabled while a user still needs them. To increment the use count, drivers
58  * call drm_crtc_vblank_get() and release the vblank reference again with
59  * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
60  * guaranteed to be enabled.
61  *
62  * On many hardware disabling the vblank interrupt cannot be done in a race-free
63  * manner, see &drm_driver.vblank_disable_immediate and
64  * &drm_driver.max_vblank_count. In that case the vblank core only disables the
65  * vblanks after a timer has expired, which can be configured through the
66  * ``vblankoffdelay`` module parameter.
67  */
68 
69 /* Retry timestamp calculation up to 3 times to satisfy
70  * drm_timestamp_precision before giving up.
71  */
72 #define DRM_TIMESTAMP_MAXRETRIES 3
73 
74 /* Threshold in nanoseconds for detection of redundant
75  * vblank irq in drm_handle_vblank(). 1 msec should be ok.
76  */
77 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
78 
79 static bool
80 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
81 			  ktime_t *tvblank, bool in_vblank_irq);
82 
83 unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
84 
85 int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
86 
87 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
88 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
89 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
90 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
91 
store_vblank(struct drm_device * dev,unsigned int pipe,u32 vblank_count_inc,ktime_t t_vblank,u32 last)92 static void store_vblank(struct drm_device *dev, unsigned int pipe,
93 			 u32 vblank_count_inc,
94 			 ktime_t t_vblank, u32 last)
95 {
96 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
97 
98 	assert_spin_locked(&dev->vblank_time_lock);
99 
100 	vblank->last = last;
101 
102 	write_seqlock(&vblank->seqlock);
103 	vblank->time = t_vblank;
104 	vblank->count += vblank_count_inc;
105 	write_sequnlock(&vblank->seqlock);
106 }
107 
108 /*
109  * "No hw counter" fallback implementation of .get_vblank_counter() hook,
110  * if there is no useable hardware frame counter available.
111  */
drm_vblank_no_hw_counter(struct drm_device * dev,unsigned int pipe)112 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
113 {
114 	WARN_ON_ONCE(dev->max_vblank_count != 0);
115 	return 0;
116 }
117 
__get_vblank_counter(struct drm_device * dev,unsigned int pipe)118 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
119 {
120 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
121 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
122 
123 		if (crtc->funcs->get_vblank_counter)
124 			return crtc->funcs->get_vblank_counter(crtc);
125 	}
126 
127 	if (dev->driver->get_vblank_counter)
128 		return dev->driver->get_vblank_counter(dev, pipe);
129 
130 	return drm_vblank_no_hw_counter(dev, pipe);
131 }
132 
133 /*
134  * Reset the stored timestamp for the current vblank count to correspond
135  * to the last vblank occurred.
136  *
137  * Only to be called from drm_crtc_vblank_on().
138  *
139  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
140  * device vblank fields.
141  */
drm_reset_vblank_timestamp(struct drm_device * dev,unsigned int pipe)142 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
143 {
144 	u32 cur_vblank;
145 	bool rc;
146 	ktime_t t_vblank;
147 	int count = DRM_TIMESTAMP_MAXRETRIES;
148 
149 	lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE);
150 
151 	/*
152 	 * sample the current counter to avoid random jumps
153 	 * when drm_vblank_enable() applies the diff
154 	 */
155 	do {
156 		cur_vblank = __get_vblank_counter(dev, pipe);
157 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
158 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
159 
160 	/*
161 	 * Only reinitialize corresponding vblank timestamp if high-precision query
162 	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
163 	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
164 	 */
165 	if (!rc)
166 		t_vblank = 0;
167 
168 	/*
169 	 * +1 to make sure user will never see the same
170 	 * vblank counter value before and after a modeset
171 	 */
172 	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
173 
174 	lockmgr(&dev->vblank_time_lock, LK_RELEASE);
175 }
176 
177 /*
178  * Call back into the driver to update the appropriate vblank counter
179  * (specified by @pipe).  Deal with wraparound, if it occurred, and
180  * update the last read value so we can deal with wraparound on the next
181  * call if necessary.
182  *
183  * Only necessary when going from off->on, to account for frames we
184  * didn't get an interrupt for.
185  *
186  * Note: caller must hold &drm_device.vbl_lock since this reads & writes
187  * device vblank fields.
188  */
drm_update_vblank_count(struct drm_device * dev,unsigned int pipe,bool in_vblank_irq)189 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
190 				    bool in_vblank_irq)
191 {
192 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
193 	u32 cur_vblank, diff;
194 	bool rc;
195 	ktime_t t_vblank;
196 	int count = DRM_TIMESTAMP_MAXRETRIES;
197 	int framedur_ns = vblank->framedur_ns;
198 
199 	/*
200 	 * Interrupts were disabled prior to this call, so deal with counter
201 	 * wrap if needed.
202 	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
203 	 * here if the register is small or we had vblank interrupts off for
204 	 * a long time.
205 	 *
206 	 * We repeat the hardware vblank counter & timestamp query until
207 	 * we get consistent results. This to prevent races between gpu
208 	 * updating its hardware counter while we are retrieving the
209 	 * corresponding vblank timestamp.
210 	 */
211 	do {
212 		cur_vblank = __get_vblank_counter(dev, pipe);
213 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
214 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
215 
216 	if (dev->max_vblank_count != 0) {
217 		/* trust the hw counter when it's around */
218 		diff = (cur_vblank - vblank->last) & dev->max_vblank_count;
219 	} else if (rc && framedur_ns) {
220 		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
221 
222 		/*
223 		 * Figure out how many vblanks we've missed based
224 		 * on the difference in the timestamps and the
225 		 * frame/field duration.
226 		 */
227 		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
228 
229 		if (diff == 0 && in_vblank_irq)
230 			DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored."
231 				      " diff_ns = %lld, framedur_ns = %d)\n",
232 				      pipe, (long long) diff_ns, framedur_ns);
233 	} else {
234 		/* some kind of default for drivers w/o accurate vbl timestamping */
235 		diff = in_vblank_irq ? 1 : 0;
236 	}
237 
238 	/*
239 	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
240 	 * interval? If so then vblank irqs keep running and it will likely
241 	 * happen that the hardware vblank counter is not trustworthy as it
242 	 * might reset at some point in that interval and vblank timestamps
243 	 * are not trustworthy either in that interval. Iow. this can result
244 	 * in a bogus diff >> 1 which must be avoided as it would cause
245 	 * random large forward jumps of the software vblank counter.
246 	 */
247 	if (diff > 1 && (vblank->inmodeset & 0x2)) {
248 		DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
249 			      " due to pre-modeset.\n", pipe, diff);
250 		diff = 1;
251 	}
252 
253 	DRM_DEBUG_VBL("updating vblank count on crtc %u:"
254 		      " current=%llu, diff=%u, hw=%u hw_last=%u\n",
255 		      pipe, vblank->count, diff, cur_vblank, vblank->last);
256 
257 	if (diff == 0) {
258 		WARN_ON_ONCE(cur_vblank != vblank->last);
259 		return;
260 	}
261 
262 	/*
263 	 * Only reinitialize corresponding vblank timestamp if high-precision query
264 	 * available and didn't fail, or we were called from the vblank interrupt.
265 	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
266 	 * for now, to mark the vblanktimestamp as invalid.
267 	 */
268 	if (!rc && !in_vblank_irq)
269 		t_vblank = 0;
270 
271 	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
272 }
273 
drm_vblank_count(struct drm_device * dev,unsigned int pipe)274 static u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
275 {
276 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
277 
278 	if (WARN_ON(pipe >= dev->num_crtcs))
279 		return 0;
280 
281 	return vblank->count;
282 }
283 
284 /**
285  * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
286  * @crtc: which counter to retrieve
287  *
288  * This function is similar to drm_crtc_vblank_count() but this function
289  * interpolates to handle a race with vblank interrupts using the high precision
290  * timestamping support.
291  *
292  * This is mostly useful for hardware that can obtain the scanout position, but
293  * doesn't have a hardware frame counter.
294  */
drm_crtc_accurate_vblank_count(struct drm_crtc * crtc)295 u32 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
296 {
297 	struct drm_device *dev = crtc->dev;
298 	unsigned int pipe = drm_crtc_index(crtc);
299 	u32 vblank;
300 	unsigned long flags;
301 
302 	WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
303 		  "This function requires support for accurate vblank timestamps.");
304 
305 	spin_lock_irqsave(&dev->vblank_time_lock, flags);
306 
307 	drm_update_vblank_count(dev, pipe, false);
308 	vblank = drm_vblank_count(dev, pipe);
309 
310 	spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
311 
312 	return vblank;
313 }
314 EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
315 
__disable_vblank(struct drm_device * dev,unsigned int pipe)316 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
317 {
318 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
319 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
320 
321 		if (crtc->funcs->disable_vblank) {
322 			crtc->funcs->disable_vblank(crtc);
323 			return;
324 		}
325 	}
326 
327 	dev->driver->disable_vblank(dev, pipe);
328 }
329 
330 /*
331  * Disable vblank irq's on crtc, make sure that last vblank count
332  * of hardware and corresponding consistent software vblank counter
333  * are preserved, even if there are any spurious vblank irq's after
334  * disable.
335  */
drm_vblank_disable_and_save(struct drm_device * dev,unsigned int pipe)336 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
337 {
338 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
339 	unsigned long irqflags;
340 
341 	assert_spin_locked(&dev->vbl_lock);
342 
343 	/* Prevent vblank irq processing while disabling vblank irqs,
344 	 * so no updates of timestamps or count can happen after we've
345 	 * disabled. Needed to prevent races in case of delayed irq's.
346 	 */
347 	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
348 
349 	/*
350 	 * Only disable vblank interrupts if they're enabled. This avoids
351 	 * calling the ->disable_vblank() operation in atomic context with the
352 	 * hardware potentially runtime suspended.
353 	 */
354 	if (vblank->enabled) {
355 		__disable_vblank(dev, pipe);
356 		vblank->enabled = false;
357 	}
358 
359 	/*
360 	 * Always update the count and timestamp to maintain the
361 	 * appearance that the counter has been ticking all along until
362 	 * this time. This makes the count account for the entire time
363 	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
364 	 */
365 	drm_update_vblank_count(dev, pipe, false);
366 
367 	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
368 }
369 
vblank_disable_fn(struct timer_list * t)370 static void vblank_disable_fn(struct timer_list *t)
371 {
372 	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
373 	struct drm_device *dev = vblank->dev;
374 	unsigned int pipe = vblank->pipe;
375 	unsigned long irqflags;
376 
377 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
378 	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
379 		DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
380 		drm_vblank_disable_and_save(dev, pipe);
381 	}
382 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
383 }
384 
drm_vblank_cleanup(struct drm_device * dev)385 void drm_vblank_cleanup(struct drm_device *dev)
386 {
387 	unsigned int pipe;
388 
389 	/* Bail if the driver didn't call drm_vblank_init() */
390 	if (dev->num_crtcs == 0)
391 		return;
392 
393 	for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
394 		struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
395 
396 		WARN_ON(READ_ONCE(vblank->enabled) &&
397 			drm_core_check_feature(dev, DRIVER_MODESET));
398 
399 		del_timer_sync(&vblank->disable_timer);
400 	}
401 
402 	kfree(dev->vblank);
403 
404 	dev->num_crtcs = 0;
405 }
406 
407 /**
408  * drm_vblank_init - initialize vblank support
409  * @dev: DRM device
410  * @num_crtcs: number of CRTCs supported by @dev
411  *
412  * This function initializes vblank support for @num_crtcs display pipelines.
413  * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
414  * drivers with a &drm_driver.release callback.
415  *
416  * Returns:
417  * Zero on success or a negative error code on failure.
418  */
drm_vblank_init(struct drm_device * dev,unsigned int num_crtcs)419 int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
420 {
421 	int ret = -ENOMEM;
422 	unsigned int i;
423 
424 	lockinit(&dev->vbl_lock, "drmvbl", 0, 0);
425 	lockinit(&dev->vblank_time_lock, "drmvtl", 0, 0);
426 
427 	dev->num_crtcs = num_crtcs;
428 
429 	dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
430 	if (!dev->vblank)
431 		goto err;
432 
433 	for (i = 0; i < num_crtcs; i++) {
434 		struct drm_vblank_crtc *vblank = &dev->vblank[i];
435 
436 		vblank->dev = dev;
437 		vblank->pipe = i;
438 		init_waitqueue_head(&vblank->queue);
439 		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
440 		seqlock_init(&vblank->seqlock);
441 	}
442 
443 	DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
444 
445 	/* Driver specific high-precision vblank timestamping supported? */
446 	if (dev->driver->get_vblank_timestamp)
447 		DRM_INFO("Driver supports precise vblank timestamp query.\n");
448 	else
449 		DRM_INFO("No driver support for vblank timestamp query.\n");
450 
451 	/* Must have precise timestamping for reliable vblank instant disable */
452 	if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
453 		dev->vblank_disable_immediate = false;
454 		DRM_INFO("Setting vblank_disable_immediate to false because "
455 			 "get_vblank_timestamp == NULL\n");
456 	}
457 
458 	return 0;
459 
460 err:
461 	dev->num_crtcs = 0;
462 	return ret;
463 }
464 EXPORT_SYMBOL(drm_vblank_init);
465 
466 /**
467  * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
468  * @crtc: which CRTC's vblank waitqueue to retrieve
469  *
470  * This function returns a pointer to the vblank waitqueue for the CRTC.
471  * Drivers can use this to implement vblank waits using wait_event() and related
472  * functions.
473  */
drm_crtc_vblank_waitqueue(struct drm_crtc * crtc)474 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
475 {
476 	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
477 }
478 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
479 
480 
481 /**
482  * drm_calc_timestamping_constants - calculate vblank timestamp constants
483  * @crtc: drm_crtc whose timestamp constants should be updated.
484  * @mode: display mode containing the scanout timings
485  *
486  * Calculate and store various constants which are later needed by vblank and
487  * swap-completion timestamping, e.g, by
488  * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
489  * scanout timing, so they take things like panel scaling or other adjustments
490  * into account.
491  */
drm_calc_timestamping_constants(struct drm_crtc * crtc,const struct drm_display_mode * mode)492 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
493 				     const struct drm_display_mode *mode)
494 {
495 	struct drm_device *dev = crtc->dev;
496 	unsigned int pipe = drm_crtc_index(crtc);
497 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
498 	int linedur_ns = 0, framedur_ns = 0;
499 	int dotclock = mode->crtc_clock;
500 
501 	if (!dev->num_crtcs)
502 		return;
503 
504 	if (WARN_ON(pipe >= dev->num_crtcs))
505 		return;
506 
507 	/* Valid dotclock? */
508 	if (dotclock > 0) {
509 		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
510 
511 		/*
512 		 * Convert scanline length in pixels and video
513 		 * dot clock to line duration and frame duration
514 		 * in nanoseconds:
515 		 */
516 		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
517 		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
518 
519 		/*
520 		 * Fields of interlaced scanout modes are only half a frame duration.
521 		 */
522 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
523 			framedur_ns /= 2;
524 	} else
525 		DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
526 			  crtc->base.id);
527 
528 	vblank->linedur_ns  = linedur_ns;
529 	vblank->framedur_ns = framedur_ns;
530 	vblank->hwmode = *mode;
531 
532 	DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
533 		  crtc->base.id, mode->crtc_htotal,
534 		  mode->crtc_vtotal, mode->crtc_vdisplay);
535 	DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
536 		  crtc->base.id, dotclock, framedur_ns, linedur_ns);
537 }
538 EXPORT_SYMBOL(drm_calc_timestamping_constants);
539 
540 /**
541  * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
542  * @dev: DRM device
543  * @pipe: index of CRTC whose vblank timestamp to retrieve
544  * @max_error: Desired maximum allowable error in timestamps (nanosecs)
545  *             On return contains true maximum error of timestamp
546  * @vblank_time: Pointer to time which should receive the timestamp
547  * @in_vblank_irq:
548  *     True when called from drm_crtc_handle_vblank().  Some drivers
549  *     need to apply some workarounds for gpu-specific vblank irq quirks
550  *     if flag is set.
551  *
552  * Implements calculation of exact vblank timestamps from given drm_display_mode
553  * timings and current video scanout position of a CRTC. This can be directly
554  * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
555  * if &drm_driver.get_scanout_position is implemented.
556  *
557  * The current implementation only handles standard video modes. For double scan
558  * and interlaced modes the driver is supposed to adjust the hardware mode
559  * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
560  * match the scanout position reported.
561  *
562  * Note that atomic drivers must call drm_calc_timestamping_constants() before
563  * enabling a CRTC. The atomic helpers already take care of that in
564  * drm_atomic_helper_update_legacy_modeset_state().
565  *
566  * Returns:
567  *
568  * Returns true on success, and false on failure, i.e. when no accurate
569  * timestamp could be acquired.
570  */
drm_calc_vbltimestamp_from_scanoutpos(struct drm_device * dev,unsigned int pipe,int * max_error,ktime_t * vblank_time,bool in_vblank_irq)571 bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
572 					   unsigned int pipe,
573 					   int *max_error,
574 					   ktime_t *vblank_time,
575 					   bool in_vblank_irq)
576 {
577 	struct timespec64 ts_etime, ts_vblank_time;
578 	ktime_t stime, etime;
579 	bool vbl_status;
580 	struct drm_crtc *crtc;
581 	const struct drm_display_mode *mode;
582 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
583 	int vpos, hpos, i;
584 	int delta_ns, duration_ns;
585 
586 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
587 		return false;
588 
589 	crtc = drm_crtc_from_index(dev, pipe);
590 
591 	if (pipe >= dev->num_crtcs || !crtc) {
592 		DRM_ERROR("Invalid crtc %u\n", pipe);
593 		return false;
594 	}
595 
596 	/* Scanout position query not supported? Should not happen. */
597 	if (!dev->driver->get_scanout_position) {
598 		DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
599 		return false;
600 	}
601 
602 	if (drm_drv_uses_atomic_modeset(dev))
603 		mode = &vblank->hwmode;
604 	else
605 		mode = &crtc->hwmode;
606 
607 	/* If mode timing undefined, just return as no-op:
608 	 * Happens during initial modesetting of a crtc.
609 	 */
610 	if (mode->crtc_clock == 0) {
611 		DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
612 		WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
613 
614 		return false;
615 	}
616 
617 	/* Get current scanout position with system timestamp.
618 	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
619 	 * if single query takes longer than max_error nanoseconds.
620 	 *
621 	 * This guarantees a tight bound on maximum error if
622 	 * code gets preempted or delayed for some reason.
623 	 */
624 	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
625 		/*
626 		 * Get vertical and horizontal scanout position vpos, hpos,
627 		 * and bounding timestamps stime, etime, pre/post query.
628 		 */
629 		vbl_status = dev->driver->get_scanout_position(dev, pipe,
630 							       in_vblank_irq,
631 							       &vpos, &hpos,
632 							       &stime, &etime,
633 							       mode);
634 
635 		/* Return as no-op if scanout query unsupported or failed. */
636 		if (!vbl_status) {
637 			DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
638 				  pipe);
639 			return false;
640 		}
641 
642 		/* Compute uncertainty in timestamp of scanout position query. */
643 		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
644 
645 		/* Accept result with <  max_error nsecs timing uncertainty. */
646 		if (duration_ns <= *max_error)
647 			break;
648 	}
649 
650 	/* Noisy system timing? */
651 	if (i == DRM_TIMESTAMP_MAXRETRIES) {
652 		DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
653 			  pipe, duration_ns/1000, *max_error/1000, i);
654 	}
655 
656 	/* Return upper bound of timestamp precision error. */
657 	*max_error = duration_ns;
658 
659 	/* Convert scanout position into elapsed time at raw_time query
660 	 * since start of scanout at first display scanline. delta_ns
661 	 * can be negative if start of scanout hasn't happened yet.
662 	 */
663 	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
664 			   mode->crtc_clock);
665 
666 	/* Subtract time delta from raw timestamp to get final
667 	 * vblank_time timestamp for end of vblank.
668 	 */
669 	*vblank_time = ktime_sub_ns(etime, delta_ns);
670 
671 	if ((drm_debug & DRM_UT_VBL) == 0)
672 		return true;
673 
674 	ts_etime = ktime_to_timespec64(etime);
675 	ts_vblank_time = ktime_to_timespec64(*vblank_time);
676 
677 	DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
678 		      pipe, hpos, vpos,
679 		      (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
680 		      (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
681 		      duration_ns / 1000, i);
682 
683 	return true;
684 }
685 EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
686 
687 /**
688  * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
689  *                             vblank interval
690  * @dev: DRM device
691  * @pipe: index of CRTC whose vblank timestamp to retrieve
692  * @tvblank: Pointer to target time which should receive the timestamp
693  * @in_vblank_irq:
694  *     True when called from drm_crtc_handle_vblank().  Some drivers
695  *     need to apply some workarounds for gpu-specific vblank irq quirks
696  *     if flag is set.
697  *
698  * Fetches the system timestamp corresponding to the time of the most recent
699  * vblank interval on specified CRTC. May call into kms-driver to
700  * compute the timestamp with a high-precision GPU specific method.
701  *
702  * Returns zero if timestamp originates from uncorrected do_gettimeofday()
703  * call, i.e., it isn't very precisely locked to the true vblank.
704  *
705  * Returns:
706  * True if timestamp is considered to be very precise, false otherwise.
707  */
708 static bool
drm_get_last_vbltimestamp(struct drm_device * dev,unsigned int pipe,ktime_t * tvblank,bool in_vblank_irq)709 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
710 			  ktime_t *tvblank, bool in_vblank_irq)
711 {
712 	bool ret = false;
713 
714 	/* Define requested maximum error on timestamps (nanoseconds). */
715 	int max_error = (int) drm_timestamp_precision * 1000;
716 
717 	/* Query driver if possible and precision timestamping enabled. */
718 	if (dev->driver->get_vblank_timestamp && (max_error > 0))
719 		ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
720 							tvblank, in_vblank_irq);
721 
722 	/* GPU high precision timestamp query unsupported or failed.
723 	 * Return current monotonic/gettimeofday timestamp as best estimate.
724 	 */
725 	if (!ret)
726 		*tvblank = ktime_get();
727 
728 	return ret;
729 }
730 
731 /**
732  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
733  * @crtc: which counter to retrieve
734  *
735  * Fetches the "cooked" vblank count value that represents the number of
736  * vblank events since the system was booted, including lost events due to
737  * modesetting activity. Note that this timer isn't correct against a racing
738  * vblank interrupt (since it only reports the software vblank counter), see
739  * drm_crtc_accurate_vblank_count() for such use-cases.
740  *
741  * Returns:
742  * The software vblank counter.
743  */
drm_crtc_vblank_count(struct drm_crtc * crtc)744 u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
745 {
746 	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
747 }
748 EXPORT_SYMBOL(drm_crtc_vblank_count);
749 
750 /**
751  * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
752  *     system timestamp corresponding to that vblank counter value.
753  * @dev: DRM device
754  * @pipe: index of CRTC whose counter to retrieve
755  * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
756  *
757  * Fetches the "cooked" vblank count value that represents the number of
758  * vblank events since the system was booted, including lost events due to
759  * modesetting activity. Returns corresponding system timestamp of the time
760  * of the vblank interval that corresponds to the current vblank counter value.
761  *
762  * This is the legacy version of drm_crtc_vblank_count_and_time().
763  */
drm_vblank_count_and_time(struct drm_device * dev,unsigned int pipe,ktime_t * vblanktime)764 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
765 				     ktime_t *vblanktime)
766 {
767 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
768 	u64 vblank_count;
769 	unsigned int seq;
770 
771 	if (WARN_ON(pipe >= dev->num_crtcs)) {
772 		*vblanktime = 0;
773 		return 0;
774 	}
775 
776 	do {
777 		seq = read_seqbegin(&vblank->seqlock);
778 		vblank_count = vblank->count;
779 		*vblanktime = vblank->time;
780 	} while (read_seqretry(&vblank->seqlock, seq));
781 
782 	return vblank_count;
783 }
784 
785 /**
786  * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
787  *     and the system timestamp corresponding to that vblank counter value
788  * @crtc: which counter to retrieve
789  * @vblanktime: Pointer to time to receive the vblank timestamp.
790  *
791  * Fetches the "cooked" vblank count value that represents the number of
792  * vblank events since the system was booted, including lost events due to
793  * modesetting activity. Returns corresponding system timestamp of the time
794  * of the vblank interval that corresponds to the current vblank counter value.
795  */
drm_crtc_vblank_count_and_time(struct drm_crtc * crtc,ktime_t * vblanktime)796 u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
797 				   ktime_t *vblanktime)
798 {
799 	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
800 					 vblanktime);
801 }
802 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
803 
send_vblank_event(struct drm_device * dev,struct drm_pending_vblank_event * e,u64 seq,ktime_t now)804 static void send_vblank_event(struct drm_device *dev,
805 		struct drm_pending_vblank_event *e,
806 		u64 seq, ktime_t now)
807 {
808 	struct timespec64 tv;
809 
810 	switch (e->event.base.type) {
811 	case DRM_EVENT_VBLANK:
812 	case DRM_EVENT_FLIP_COMPLETE:
813 		tv = ktime_to_timespec64(now);
814 		e->event.vbl.sequence = seq;
815 		/*
816 		 * e->event is a user space structure, with hardcoded unsigned
817 		 * 32-bit seconds/microseconds. This is safe as we always use
818 		 * monotonic timestamps since linux-4.15
819 		 */
820 		e->event.vbl.tv_sec = tv.tv_sec;
821 		e->event.vbl.tv_usec = tv.tv_nsec / 1000;
822 		break;
823 	case DRM_EVENT_CRTC_SEQUENCE:
824 		if (seq)
825 			e->event.seq.sequence = seq;
826 		e->event.seq.time_ns = ktime_to_ns(now);
827 		break;
828 	}
829 	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
830 	drm_send_event_locked(dev, &e->base);
831 }
832 
833 /**
834  * drm_crtc_arm_vblank_event - arm vblank event after pageflip
835  * @crtc: the source CRTC of the vblank event
836  * @e: the event to send
837  *
838  * A lot of drivers need to generate vblank events for the very next vblank
839  * interrupt. For example when the page flip interrupt happens when the page
840  * flip gets armed, but not when it actually executes within the next vblank
841  * period. This helper function implements exactly the required vblank arming
842  * behaviour.
843  *
844  * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
845  * atomic commit must ensure that the next vblank happens at exactly the same
846  * time as the atomic commit is committed to the hardware. This function itself
847  * does **not** protect against the next vblank interrupt racing with either this
848  * function call or the atomic commit operation. A possible sequence could be:
849  *
850  * 1. Driver commits new hardware state into vblank-synchronized registers.
851  * 2. A vblank happens, committing the hardware state. Also the corresponding
852  *    vblank interrupt is fired off and fully processed by the interrupt
853  *    handler.
854  * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
855  * 4. The event is only send out for the next vblank, which is wrong.
856  *
857  * An equivalent race can happen when the driver calls
858  * drm_crtc_arm_vblank_event() before writing out the new hardware state.
859  *
860  * The only way to make this work safely is to prevent the vblank from firing
861  * (and the hardware from committing anything else) until the entire atomic
862  * commit sequence has run to completion. If the hardware does not have such a
863  * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
864  * Instead drivers need to manually send out the event from their interrupt
865  * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
866  * possible race with the hardware committing the atomic update.
867  *
868  * Caller must hold a vblank reference for the event @e, which will be dropped
869  * when the next vblank arrives.
870  */
drm_crtc_arm_vblank_event(struct drm_crtc * crtc,struct drm_pending_vblank_event * e)871 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
872 			       struct drm_pending_vblank_event *e)
873 {
874 	struct drm_device *dev = crtc->dev;
875 	unsigned int pipe = drm_crtc_index(crtc);
876 
877 	assert_spin_locked(&dev->event_lock);
878 
879 	e->pipe = pipe;
880 	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
881 	list_add_tail(&e->base.link, &dev->vblank_event_list);
882 }
883 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
884 
885 /**
886  * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
887  * @crtc: the source CRTC of the vblank event
888  * @e: the event to send
889  *
890  * Updates sequence # and timestamp on event for the most recently processed
891  * vblank, and sends it to userspace.  Caller must hold event lock.
892  *
893  * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
894  * situation, especially to send out events for atomic commit operations.
895  */
drm_crtc_send_vblank_event(struct drm_crtc * crtc,struct drm_pending_vblank_event * e)896 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
897 				struct drm_pending_vblank_event *e)
898 {
899 	struct drm_device *dev = crtc->dev;
900 	u64 seq;
901 	unsigned int pipe = drm_crtc_index(crtc);
902 	ktime_t now;
903 
904 	if (dev->num_crtcs > 0) {
905 		seq = drm_vblank_count_and_time(dev, pipe, &now);
906 	} else {
907 		seq = 0;
908 
909 		now = ktime_get();
910 	}
911 	e->pipe = pipe;
912 	send_vblank_event(dev, e, seq, now);
913 }
914 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
915 
__enable_vblank(struct drm_device * dev,unsigned int pipe)916 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
917 {
918 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
919 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
920 
921 		if (crtc->funcs->enable_vblank)
922 			return crtc->funcs->enable_vblank(crtc);
923 	}
924 
925 	return dev->driver->enable_vblank(dev, pipe);
926 }
927 
drm_vblank_enable(struct drm_device * dev,unsigned int pipe)928 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
929 {
930 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
931 	int ret = 0;
932 
933 	assert_spin_locked(&dev->vbl_lock);
934 
935 	lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE);
936 
937 	if (!vblank->enabled) {
938 		/*
939 		 * Enable vblank irqs under vblank_time_lock protection.
940 		 * All vblank count & timestamp updates are held off
941 		 * until we are done reinitializing master counter and
942 		 * timestamps. Filtercode in drm_handle_vblank() will
943 		 * prevent double-accounting of same vblank interval.
944 		 */
945 		ret = __enable_vblank(dev, pipe);
946 		DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
947 		if (ret) {
948 			atomic_dec(&vblank->refcount);
949 		} else {
950 			drm_update_vblank_count(dev, pipe, 0);
951 			/* drm_update_vblank_count() includes a wmb so we just
952 			 * need to ensure that the compiler emits the write
953 			 * to mark the vblank as enabled after the call
954 			 * to drm_update_vblank_count().
955 			 */
956 			WRITE_ONCE(vblank->enabled, true);
957 		}
958 	}
959 
960 	lockmgr(&dev->vblank_time_lock, LK_RELEASE);
961 
962 	return ret;
963 }
964 
drm_vblank_get(struct drm_device * dev,unsigned int pipe)965 static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
966 {
967 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
968 	unsigned long irqflags;
969 	int ret = 0;
970 
971 	if (!dev->num_crtcs)
972 		return -EINVAL;
973 
974 	if (WARN_ON(pipe >= dev->num_crtcs))
975 		return -EINVAL;
976 
977 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
978 	/* Going from 0->1 means we have to enable interrupts again */
979 	if (atomic_add_return(1, &vblank->refcount) == 1) {
980 		ret = drm_vblank_enable(dev, pipe);
981 	} else {
982 		if (!vblank->enabled) {
983 			atomic_dec(&vblank->refcount);
984 			ret = -EINVAL;
985 		}
986 	}
987 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
988 
989 	return ret;
990 }
991 
992 /**
993  * drm_crtc_vblank_get - get a reference count on vblank events
994  * @crtc: which CRTC to own
995  *
996  * Acquire a reference count on vblank events to avoid having them disabled
997  * while in use.
998  *
999  * Returns:
1000  * Zero on success or a negative error code on failure.
1001  */
drm_crtc_vblank_get(struct drm_crtc * crtc)1002 int drm_crtc_vblank_get(struct drm_crtc *crtc)
1003 {
1004 	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1005 }
1006 EXPORT_SYMBOL(drm_crtc_vblank_get);
1007 
drm_vblank_put(struct drm_device * dev,unsigned int pipe)1008 static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1009 {
1010 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1011 
1012 	if (WARN_ON(pipe >= dev->num_crtcs))
1013 		return;
1014 
1015 	if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1016 		return;
1017 
1018 	/* Last user schedules interrupt disable */
1019 	if (atomic_dec_and_test(&vblank->refcount)) {
1020 		if (drm_vblank_offdelay == 0)
1021 			return;
1022 		else if (drm_vblank_offdelay < 0)
1023 			vblank_disable_fn(&vblank->disable_timer);
1024 		else if (!dev->vblank_disable_immediate)
1025 			mod_timer(&vblank->disable_timer,
1026 				  jiffies + ((drm_vblank_offdelay * HZ)/1000));
1027 	}
1028 }
1029 
1030 /**
1031  * drm_crtc_vblank_put - give up ownership of vblank events
1032  * @crtc: which counter to give up
1033  *
1034  * Release ownership of a given vblank counter, turning off interrupts
1035  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1036  */
drm_crtc_vblank_put(struct drm_crtc * crtc)1037 void drm_crtc_vblank_put(struct drm_crtc *crtc)
1038 {
1039 	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1040 }
1041 EXPORT_SYMBOL(drm_crtc_vblank_put);
1042 
1043 /**
1044  * drm_wait_one_vblank - wait for one vblank
1045  * @dev: DRM device
1046  * @pipe: CRTC index
1047  *
1048  * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1049  * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1050  * due to lack of driver support or because the crtc is off.
1051  *
1052  * This is the legacy version of drm_crtc_wait_one_vblank().
1053  */
drm_wait_one_vblank(struct drm_device * dev,unsigned int pipe)1054 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1055 {
1056 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1057 	int ret;
1058 	u32 last;
1059 
1060 	if (WARN_ON(pipe >= dev->num_crtcs))
1061 		return;
1062 
1063 	ret = drm_vblank_get(dev, pipe);
1064 	if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1065 		return;
1066 
1067 	last = drm_vblank_count(dev, pipe);
1068 
1069 	ret = wait_event_timeout(vblank->queue,
1070 				 last != drm_vblank_count(dev, pipe),
1071 				 msecs_to_jiffies(100));
1072 
1073 	WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1074 
1075 	drm_vblank_put(dev, pipe);
1076 }
1077 EXPORT_SYMBOL(drm_wait_one_vblank);
1078 
1079 /**
1080  * drm_crtc_wait_one_vblank - wait for one vblank
1081  * @crtc: DRM crtc
1082  *
1083  * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1084  * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1085  * due to lack of driver support or because the crtc is off.
1086  */
drm_crtc_wait_one_vblank(struct drm_crtc * crtc)1087 void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1088 {
1089 	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1090 }
1091 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1092 
1093 /**
1094  * drm_crtc_vblank_off - disable vblank events on a CRTC
1095  * @crtc: CRTC in question
1096  *
1097  * Drivers can use this function to shut down the vblank interrupt handling when
1098  * disabling a crtc. This function ensures that the latest vblank frame count is
1099  * stored so that drm_vblank_on can restore it again.
1100  *
1101  * Drivers must use this function when the hardware vblank counter can get
1102  * reset, e.g. when suspending or disabling the @crtc in general.
1103  */
drm_crtc_vblank_off(struct drm_crtc * crtc)1104 void drm_crtc_vblank_off(struct drm_crtc *crtc)
1105 {
1106 	struct drm_device *dev = crtc->dev;
1107 	unsigned int pipe = drm_crtc_index(crtc);
1108 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1109 	struct drm_pending_vblank_event *e, *t;
1110 
1111 	ktime_t now;
1112 	unsigned long irqflags;
1113 	u64 seq;
1114 
1115 	if (WARN_ON(pipe >= dev->num_crtcs))
1116 		return;
1117 
1118 	spin_lock_irqsave(&dev->event_lock, irqflags);
1119 
1120 	lockmgr(&dev->vbl_lock, LK_EXCLUSIVE);
1121 	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1122 		      pipe, vblank->enabled, vblank->inmodeset);
1123 
1124 	/* Avoid redundant vblank disables without previous
1125 	 * drm_crtc_vblank_on(). */
1126 	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1127 		drm_vblank_disable_and_save(dev, pipe);
1128 
1129 	wake_up(&vblank->queue);
1130 
1131 	/*
1132 	 * Prevent subsequent drm_vblank_get() from re-enabling
1133 	 * the vblank interrupt by bumping the refcount.
1134 	 */
1135 	if (!vblank->inmodeset) {
1136 		atomic_inc(&vblank->refcount);
1137 		vblank->inmodeset = 1;
1138 	}
1139 	lockmgr(&dev->vbl_lock, LK_RELEASE);
1140 
1141 	/* Send any queued vblank events, lest the natives grow disquiet */
1142 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1143 
1144 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1145 		if (e->pipe != pipe)
1146 			continue;
1147 		DRM_DEBUG("Sending premature vblank event on disable: "
1148 			  "wanted %llu, current %llu\n",
1149 			  e->sequence, seq);
1150 		list_del(&e->base.link);
1151 		drm_vblank_put(dev, pipe);
1152 		send_vblank_event(dev, e, seq, now);
1153 	}
1154 	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1155 
1156 	/* Will be reset by the modeset helpers when re-enabling the crtc by
1157 	 * calling drm_calc_timestamping_constants(). */
1158 	vblank->hwmode.crtc_clock = 0;
1159 }
1160 EXPORT_SYMBOL(drm_crtc_vblank_off);
1161 
1162 /**
1163  * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1164  * @crtc: CRTC in question
1165  *
1166  * Drivers can use this function to reset the vblank state to off at load time.
1167  * Drivers should use this together with the drm_crtc_vblank_off() and
1168  * drm_crtc_vblank_on() functions. The difference compared to
1169  * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1170  * and hence doesn't need to call any driver hooks.
1171  *
1172  * This is useful for recovering driver state e.g. on driver load, or on resume.
1173  */
drm_crtc_vblank_reset(struct drm_crtc * crtc)1174 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1175 {
1176 	struct drm_device *dev = crtc->dev;
1177 	unsigned long irqflags;
1178 	unsigned int pipe = drm_crtc_index(crtc);
1179 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1180 
1181 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1182 	/*
1183 	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1184 	 * interrupt by bumping the refcount.
1185 	 */
1186 	if (!vblank->inmodeset) {
1187 		atomic_inc(&vblank->refcount);
1188 		vblank->inmodeset = 1;
1189 	}
1190 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1191 
1192 	WARN_ON(!list_empty(&dev->vblank_event_list));
1193 }
1194 EXPORT_SYMBOL(drm_crtc_vblank_reset);
1195 
1196 /**
1197  * drm_crtc_vblank_on - enable vblank events on a CRTC
1198  * @crtc: CRTC in question
1199  *
1200  * This functions restores the vblank interrupt state captured with
1201  * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1202  * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1203  * unbalanced and so can also be unconditionally called in driver load code to
1204  * reflect the current hardware state of the crtc.
1205  */
drm_crtc_vblank_on(struct drm_crtc * crtc)1206 void drm_crtc_vblank_on(struct drm_crtc *crtc)
1207 {
1208 	struct drm_device *dev = crtc->dev;
1209 	unsigned int pipe = drm_crtc_index(crtc);
1210 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1211 	unsigned long irqflags;
1212 
1213 	if (WARN_ON(pipe >= dev->num_crtcs))
1214 		return;
1215 
1216 	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1217 	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1218 		      pipe, vblank->enabled, vblank->inmodeset);
1219 
1220 	/* Drop our private "prevent drm_vblank_get" refcount */
1221 	if (vblank->inmodeset) {
1222 		atomic_dec(&vblank->refcount);
1223 		vblank->inmodeset = 0;
1224 	}
1225 
1226 	drm_reset_vblank_timestamp(dev, pipe);
1227 
1228 	/*
1229 	 * re-enable interrupts if there are users left, or the
1230 	 * user wishes vblank interrupts to be enabled all the time.
1231 	 */
1232 	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1233 		WARN_ON(drm_vblank_enable(dev, pipe));
1234 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1235 }
1236 EXPORT_SYMBOL(drm_crtc_vblank_on);
1237 
drm_legacy_vblank_pre_modeset(struct drm_device * dev,unsigned int pipe)1238 static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1239 					  unsigned int pipe)
1240 {
1241 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1242 
1243 	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1244 	if (!dev->num_crtcs)
1245 		return;
1246 
1247 	if (WARN_ON(pipe >= dev->num_crtcs))
1248 		return;
1249 
1250 	/*
1251 	 * To avoid all the problems that might happen if interrupts
1252 	 * were enabled/disabled around or between these calls, we just
1253 	 * have the kernel take a reference on the CRTC (just once though
1254 	 * to avoid corrupting the count if multiple, mismatch calls occur),
1255 	 * so that interrupts remain enabled in the interim.
1256 	 */
1257 	if (!vblank->inmodeset) {
1258 		vblank->inmodeset = 0x1;
1259 		if (drm_vblank_get(dev, pipe) == 0)
1260 			vblank->inmodeset |= 0x2;
1261 	}
1262 }
1263 
drm_legacy_vblank_post_modeset(struct drm_device * dev,unsigned int pipe)1264 static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1265 					   unsigned int pipe)
1266 {
1267 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1268 	unsigned long irqflags;
1269 
1270 	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1271 	if (!dev->num_crtcs)
1272 		return;
1273 
1274 	if (WARN_ON(pipe >= dev->num_crtcs))
1275 		return;
1276 
1277 	if (vblank->inmodeset) {
1278 		spin_lock_irqsave(&dev->vbl_lock, irqflags);
1279 		drm_reset_vblank_timestamp(dev, pipe);
1280 		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1281 
1282 		if (vblank->inmodeset & 0x2)
1283 			drm_vblank_put(dev, pipe);
1284 
1285 		vblank->inmodeset = 0;
1286 	}
1287 }
1288 
drm_legacy_modeset_ctl_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1289 int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1290 				 struct drm_file *file_priv)
1291 {
1292 	struct drm_modeset_ctl *modeset = data;
1293 	unsigned int pipe;
1294 
1295 	/* If drm_vblank_init() hasn't been called yet, just no-op */
1296 	if (!dev->num_crtcs)
1297 		return 0;
1298 
1299 	/* KMS drivers handle this internally */
1300 	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1301 		return 0;
1302 
1303 	pipe = modeset->crtc;
1304 	if (pipe >= dev->num_crtcs)
1305 		return -EINVAL;
1306 
1307 	switch (modeset->cmd) {
1308 	case _DRM_PRE_MODESET:
1309 		drm_legacy_vblank_pre_modeset(dev, pipe);
1310 		break;
1311 	case _DRM_POST_MODESET:
1312 		drm_legacy_vblank_post_modeset(dev, pipe);
1313 		break;
1314 	default:
1315 		return -EINVAL;
1316 	}
1317 
1318 	return 0;
1319 }
1320 
vblank_passed(u64 seq,u64 ref)1321 static inline bool vblank_passed(u64 seq, u64 ref)
1322 {
1323 	return (seq - ref) <= (1 << 23);
1324 }
1325 
drm_queue_vblank_event(struct drm_device * dev,unsigned int pipe,u64 req_seq,union drm_wait_vblank * vblwait,struct drm_file * file_priv)1326 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1327 				  u64 req_seq,
1328 				  union drm_wait_vblank *vblwait,
1329 				  struct drm_file *file_priv)
1330 {
1331 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1332 	struct drm_pending_vblank_event *e;
1333 	ktime_t now;
1334 	unsigned long flags;
1335 	u64 seq;
1336 	int ret;
1337 
1338 	e = kzalloc(sizeof(*e), GFP_KERNEL);
1339 	if (e == NULL) {
1340 		ret = -ENOMEM;
1341 		goto err_put;
1342 	}
1343 
1344 	e->pipe = pipe;
1345 	e->event.base.type = DRM_EVENT_VBLANK;
1346 	e->event.base.length = sizeof(e->event.vbl);
1347 	e->event.vbl.user_data = vblwait->request.signal;
1348 	e->event.vbl.crtc_id = 0;
1349 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1350 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1351 		if (crtc)
1352 			e->event.vbl.crtc_id = crtc->base.id;
1353 	}
1354 
1355 	spin_lock_irqsave(&dev->event_lock, flags);
1356 
1357 	/*
1358 	 * drm_crtc_vblank_off() might have been called after we called
1359 	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1360 	 * vblank disable, so no need for further locking.  The reference from
1361 	 * drm_vblank_get() protects against vblank disable from another source.
1362 	 */
1363 	if (!READ_ONCE(vblank->enabled)) {
1364 		ret = -EINVAL;
1365 		goto err_unlock;
1366 	}
1367 
1368 	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1369 					    &e->event.base);
1370 
1371 	if (ret)
1372 		goto err_unlock;
1373 
1374 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1375 
1376 	DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1377 		  req_seq, seq, pipe);
1378 
1379 	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1380 
1381 	e->sequence = req_seq;
1382 	if (vblank_passed(seq, req_seq)) {
1383 		drm_vblank_put(dev, pipe);
1384 		send_vblank_event(dev, e, seq, now);
1385 		vblwait->reply.sequence = seq;
1386 	} else {
1387 		/* drm_handle_vblank_events will call drm_vblank_put */
1388 		list_add_tail(&e->base.link, &dev->vblank_event_list);
1389 		vblwait->reply.sequence = req_seq;
1390 	}
1391 
1392 	spin_unlock_irqrestore(&dev->event_lock, flags);
1393 
1394 	return 0;
1395 
1396 err_unlock:
1397 	spin_unlock_irqrestore(&dev->event_lock, flags);
1398 	kfree(e);
1399 err_put:
1400 	drm_vblank_put(dev, pipe);
1401 	return ret;
1402 }
1403 
drm_wait_vblank_is_query(union drm_wait_vblank * vblwait)1404 static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1405 {
1406 	if (vblwait->request.sequence)
1407 		return false;
1408 
1409 	return _DRM_VBLANK_RELATIVE ==
1410 		(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1411 					  _DRM_VBLANK_EVENT |
1412 					  _DRM_VBLANK_NEXTONMISS));
1413 }
1414 
1415 /*
1416  * Widen a 32-bit param to 64-bits.
1417  *
1418  * \param narrow 32-bit value (missing upper 32 bits)
1419  * \param near 64-bit value that should be 'close' to near
1420  *
1421  * This function returns a 64-bit value using the lower 32-bits from
1422  * 'narrow' and constructing the upper 32-bits so that the result is
1423  * as close as possible to 'near'.
1424  */
1425 
widen_32_to_64(u32 narrow,u64 near)1426 static u64 widen_32_to_64(u32 narrow, u64 near)
1427 {
1428 	return near + (s32) (narrow - near);
1429 }
1430 
drm_wait_vblank_reply(struct drm_device * dev,unsigned int pipe,struct drm_wait_vblank_reply * reply)1431 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1432 				  struct drm_wait_vblank_reply *reply)
1433 {
1434 	ktime_t now;
1435 	struct timespec64 ts;
1436 
1437 	/*
1438 	 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1439 	 * to store the seconds. This is safe as we always use monotonic
1440 	 * timestamps since linux-4.15.
1441 	 */
1442 	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1443 	ts = ktime_to_timespec64(now);
1444 	reply->tval_sec = (u32)ts.tv_sec;
1445 	reply->tval_usec = ts.tv_nsec / 1000;
1446 }
1447 
drm_wait_vblank_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1448 int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1449 			  struct drm_file *file_priv)
1450 {
1451 	struct drm_crtc *crtc;
1452 	struct drm_vblank_crtc *vblank;
1453 	union drm_wait_vblank *vblwait = data;
1454 	int ret;
1455 	u64 req_seq, seq;
1456 	unsigned int pipe_index;
1457 	unsigned int flags, pipe, high_pipe;
1458 
1459 	if (!dev->irq_enabled)
1460 		return -EINVAL;
1461 
1462 	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1463 		return -EINVAL;
1464 
1465 	if (vblwait->request.type &
1466 	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1467 	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1468 		DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
1469 			  vblwait->request.type,
1470 			  (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1471 			   _DRM_VBLANK_HIGH_CRTC_MASK));
1472 		return -EINVAL;
1473 	}
1474 
1475 	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1476 	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1477 	if (high_pipe)
1478 		pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1479 	else
1480 		pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1481 
1482 	/* Convert lease-relative crtc index into global crtc index */
1483 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1484 		pipe = 0;
1485 		drm_for_each_crtc(crtc, dev) {
1486 			if (drm_lease_held(file_priv, crtc->base.id)) {
1487 				if (pipe_index == 0)
1488 					break;
1489 				pipe_index--;
1490 			}
1491 			pipe++;
1492 		}
1493 	} else {
1494 		pipe = pipe_index;
1495 	}
1496 
1497 	if (pipe >= dev->num_crtcs)
1498 		return -EINVAL;
1499 
1500 	vblank = &dev->vblank[pipe];
1501 
1502 	/* If the counter is currently enabled and accurate, short-circuit
1503 	 * queries to return the cached timestamp of the last vblank.
1504 	 */
1505 	if (dev->vblank_disable_immediate &&
1506 	    drm_wait_vblank_is_query(vblwait) &&
1507 	    READ_ONCE(vblank->enabled)) {
1508 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1509 		return 0;
1510 	}
1511 
1512 	ret = drm_vblank_get(dev, pipe);
1513 	if (ret) {
1514 		DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1515 		return ret;
1516 	}
1517 	seq = drm_vblank_count(dev, pipe);
1518 
1519 	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1520 	case _DRM_VBLANK_RELATIVE:
1521 		req_seq = seq + vblwait->request.sequence;
1522 		vblwait->request.sequence = req_seq;
1523 		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1524 		break;
1525 	case _DRM_VBLANK_ABSOLUTE:
1526 		req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1527 		break;
1528 	default:
1529 		ret = -EINVAL;
1530 		goto done;
1531 	}
1532 
1533 	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1534 	    vblank_passed(seq, req_seq)) {
1535 		req_seq = seq + 1;
1536 		vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1537 		vblwait->request.sequence = req_seq;
1538 	}
1539 
1540 	if (flags & _DRM_VBLANK_EVENT) {
1541 		/* must hold on to the vblank ref until the event fires
1542 		 * drm_vblank_put will be called asynchronously
1543 		 */
1544 		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1545 	}
1546 
1547 	if (req_seq != seq) {
1548 		DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1549 			  req_seq, pipe);
1550 		DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
1551 			    vblank_passed(drm_vblank_count(dev, pipe),
1552 					  req_seq) ||
1553 			    !READ_ONCE(vblank->enabled));
1554 	}
1555 
1556 	if (ret != -EINTR) {
1557 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1558 
1559 		DRM_DEBUG("crtc %d returning %u to client\n",
1560 			  pipe, vblwait->reply.sequence);
1561 	} else {
1562 		DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1563 	}
1564 
1565 done:
1566 	drm_vblank_put(dev, pipe);
1567 	return ret;
1568 }
1569 
drm_handle_vblank_events(struct drm_device * dev,unsigned int pipe)1570 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1571 {
1572 	struct drm_pending_vblank_event *e, *t;
1573 	ktime_t now;
1574 	u64 seq;
1575 
1576 	assert_spin_locked(&dev->event_lock);
1577 
1578 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1579 
1580 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1581 		if (e->pipe != pipe)
1582 			continue;
1583 		if (!vblank_passed(seq, e->sequence))
1584 			continue;
1585 
1586 		DRM_DEBUG("vblank event on %llu, current %llu\n",
1587 			  e->sequence, seq);
1588 
1589 		list_del(&e->base.link);
1590 		drm_vblank_put(dev, pipe);
1591 		send_vblank_event(dev, e, seq, now);
1592 	}
1593 
1594 	trace_drm_vblank_event(pipe, seq);
1595 }
1596 
1597 /**
1598  * drm_handle_vblank - handle a vblank event
1599  * @dev: DRM device
1600  * @pipe: index of CRTC where this event occurred
1601  *
1602  * Drivers should call this routine in their vblank interrupt handlers to
1603  * update the vblank counter and send any signals that may be pending.
1604  *
1605  * This is the legacy version of drm_crtc_handle_vblank().
1606  */
drm_handle_vblank(struct drm_device * dev,unsigned int pipe)1607 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1608 {
1609 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1610 	unsigned long irqflags;
1611 	bool disable_irq;
1612 
1613 	if (WARN_ON_ONCE(!dev->num_crtcs))
1614 		return false;
1615 
1616 	if (WARN_ON(pipe >= dev->num_crtcs))
1617 		return false;
1618 
1619 	spin_lock_irqsave(&dev->event_lock, irqflags);
1620 
1621 	/* Need timestamp lock to prevent concurrent execution with
1622 	 * vblank enable/disable, as this would cause inconsistent
1623 	 * or corrupted timestamps and vblank counts.
1624 	 */
1625 	lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE);
1626 
1627 	/* Vblank irq handling disabled. Nothing to do. */
1628 	if (!vblank->enabled) {
1629 		lockmgr(&dev->vblank_time_lock, LK_RELEASE);
1630 		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1631 		return false;
1632 	}
1633 
1634 	drm_update_vblank_count(dev, pipe, true);
1635 
1636 	lockmgr(&dev->vblank_time_lock, LK_RELEASE);
1637 
1638 	wake_up(&vblank->queue);
1639 
1640 	/* With instant-off, we defer disabling the interrupt until after
1641 	 * we finish processing the following vblank after all events have
1642 	 * been signaled. The disable has to be last (after
1643 	 * drm_handle_vblank_events) so that the timestamp is always accurate.
1644 	 */
1645 	disable_irq = (dev->vblank_disable_immediate &&
1646 		       drm_vblank_offdelay > 0 &&
1647 		       !atomic_read(&vblank->refcount));
1648 
1649 	drm_handle_vblank_events(dev, pipe);
1650 
1651 	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1652 
1653 	if (disable_irq)
1654 		vblank_disable_fn(&vblank->disable_timer);
1655 
1656 	return true;
1657 }
1658 EXPORT_SYMBOL(drm_handle_vblank);
1659 
1660 /**
1661  * drm_crtc_handle_vblank - handle a vblank event
1662  * @crtc: where this event occurred
1663  *
1664  * Drivers should call this routine in their vblank interrupt handlers to
1665  * update the vblank counter and send any signals that may be pending.
1666  *
1667  * This is the native KMS version of drm_handle_vblank().
1668  *
1669  * Returns:
1670  * True if the event was successfully handled, false on failure.
1671  */
drm_crtc_handle_vblank(struct drm_crtc * crtc)1672 bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1673 {
1674 	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1675 }
1676 EXPORT_SYMBOL(drm_crtc_handle_vblank);
1677 
1678 /*
1679  * Get crtc VBLANK count.
1680  *
1681  * \param dev DRM device
1682  * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1683  * \param file_priv drm file private for the user's open file descriptor
1684  */
1685 
drm_crtc_get_sequence_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1686 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1687 				struct drm_file *file_priv)
1688 {
1689 	struct drm_crtc *crtc;
1690 	struct drm_vblank_crtc *vblank;
1691 	int pipe;
1692 	struct drm_crtc_get_sequence *get_seq = data;
1693 	ktime_t now;
1694 	bool vblank_enabled;
1695 	int ret;
1696 
1697 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
1698 		return -EINVAL;
1699 
1700 	if (!dev->irq_enabled)
1701 		return -EINVAL;
1702 
1703 	crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1704 	if (!crtc)
1705 		return -ENOENT;
1706 
1707 	pipe = drm_crtc_index(crtc);
1708 
1709 	vblank = &dev->vblank[pipe];
1710 	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1711 
1712 	if (!vblank_enabled) {
1713 		ret = drm_crtc_vblank_get(crtc);
1714 		if (ret) {
1715 			DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1716 			return ret;
1717 		}
1718 	}
1719 	drm_modeset_lock(&crtc->mutex, NULL);
1720 	if (crtc->state)
1721 		get_seq->active = crtc->state->enable;
1722 	else
1723 		get_seq->active = crtc->enabled;
1724 	drm_modeset_unlock(&crtc->mutex);
1725 	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1726 	get_seq->sequence_ns = ktime_to_ns(now);
1727 	if (!vblank_enabled)
1728 		drm_crtc_vblank_put(crtc);
1729 	return 0;
1730 }
1731 
1732 /*
1733  * Queue a event for VBLANK sequence
1734  *
1735  * \param dev DRM device
1736  * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1737  * \param file_priv drm file private for the user's open file descriptor
1738  */
1739 
drm_crtc_queue_sequence_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)1740 int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1741 				  struct drm_file *file_priv)
1742 {
1743 	struct drm_crtc *crtc;
1744 	struct drm_vblank_crtc *vblank;
1745 	int pipe;
1746 	struct drm_crtc_queue_sequence *queue_seq = data;
1747 	ktime_t now;
1748 	struct drm_pending_vblank_event *e;
1749 	u32 flags;
1750 	u64 seq;
1751 	u64 req_seq;
1752 	int ret;
1753 	unsigned long spin_flags;
1754 
1755 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
1756 		return -EINVAL;
1757 
1758 	if (!dev->irq_enabled)
1759 		return -EINVAL;
1760 
1761 	crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1762 	if (!crtc)
1763 		return -ENOENT;
1764 
1765 	flags = queue_seq->flags;
1766 	/* Check valid flag bits */
1767 	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1768 		      DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1769 		return -EINVAL;
1770 
1771 	pipe = drm_crtc_index(crtc);
1772 
1773 	vblank = &dev->vblank[pipe];
1774 
1775 	e = kzalloc(sizeof(*e), GFP_KERNEL);
1776 	if (e == NULL)
1777 		return -ENOMEM;
1778 
1779 	ret = drm_crtc_vblank_get(crtc);
1780 	if (ret) {
1781 		DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1782 		goto err_free;
1783 	}
1784 
1785 	seq = drm_vblank_count_and_time(dev, pipe, &now);
1786 	req_seq = queue_seq->sequence;
1787 
1788 	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1789 		req_seq += seq;
1790 
1791 	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1792 		req_seq = seq + 1;
1793 
1794 	e->pipe = pipe;
1795 	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1796 	e->event.base.length = sizeof(e->event.seq);
1797 	e->event.seq.user_data = queue_seq->user_data;
1798 
1799 	spin_lock_irqsave(&dev->event_lock, spin_flags);
1800 
1801 	/*
1802 	 * drm_crtc_vblank_off() might have been called after we called
1803 	 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1804 	 * vblank disable, so no need for further locking.  The reference from
1805 	 * drm_crtc_vblank_get() protects against vblank disable from another source.
1806 	 */
1807 	if (!READ_ONCE(vblank->enabled)) {
1808 		ret = -EINVAL;
1809 		goto err_unlock;
1810 	}
1811 
1812 	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1813 					    &e->event.base);
1814 
1815 	if (ret)
1816 		goto err_unlock;
1817 
1818 	e->sequence = req_seq;
1819 
1820 	if (vblank_passed(seq, req_seq)) {
1821 		drm_crtc_vblank_put(crtc);
1822 		send_vblank_event(dev, e, seq, now);
1823 		queue_seq->sequence = seq;
1824 	} else {
1825 		/* drm_handle_vblank_events will call drm_vblank_put */
1826 		list_add_tail(&e->base.link, &dev->vblank_event_list);
1827 		queue_seq->sequence = req_seq;
1828 	}
1829 
1830 	spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1831 	return 0;
1832 
1833 err_unlock:
1834 	spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1835 	drm_crtc_vblank_put(crtc);
1836 err_free:
1837 	kfree(e);
1838 	return ret;
1839 }
1840