40 #define vblanktimestamp(dev, crtc, count) ( \  argument
41 	(dev)->_vblank_time[(crtc) * DRM_VBLANKTIME_RBSIZE + \
67 int drm_irq_by_busid(struct drm_device *dev, void *data,  in drm_irq_by_busid()  argument
72 	if (!dev->driver->bus->irq_by_busid)  in drm_irq_by_busid()
75 	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))  in drm_irq_by_busid()
78 	return dev->driver->bus->irq_by_busid(dev, p);  in drm_irq_by_busid()
84 static void clear_vblank_timestamps(struct drm_device *dev, int crtc)  in clear_vblank_timestamps()  argument
86 	memset(&dev->_vblank_time[crtc * DRM_VBLANKTIME_RBSIZE], 0,  in clear_vblank_timestamps()
96 static void vblank_disable_and_save(struct drm_device *dev, int crtc)  in vblank_disable_and_save()  argument
108 	mtx_lock(&dev->vblank_time_lock);  in vblank_disable_and_save()
110 	dev->driver->disable_vblank(dev, crtc);  in vblank_disable_and_save()
111 	dev->vblank_enabled[crtc] = 0;  in vblank_disable_and_save()
126 		dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);  in vblank_disable_and_save()
127 		vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);  in vblank_disable_and_save()
128 …} while (dev->last_vblank[crtc] != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblr…  in vblank_disable_and_save()
136 	vblcount = atomic_read(&dev->_vblank_count[crtc]);  in vblank_disable_and_save()
138 		  timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));  in vblank_disable_and_save()
153 		atomic_inc(&dev->_vblank_count[crtc]);  in vblank_disable_and_save()
158 	clear_vblank_timestamps(dev, crtc);  in vblank_disable_and_save()
160 	mtx_unlock(&dev->vblank_time_lock);  in vblank_disable_and_save()
165 	struct drm_device *dev = (struct drm_device *)arg;  in vblank_disable_fn()  local
168 	if (!dev->vblank_disable_allowed)  in vblank_disable_fn()
171 	for (i = 0; i < dev->num_crtcs; i++) {  in vblank_disable_fn()
172 		mtx_lock(&dev->vbl_lock);  in vblank_disable_fn()
173 		if (atomic_read(&dev->vblank_refcount[i]) == 0 &&  in vblank_disable_fn()
174 		    dev->vblank_enabled[i]) {  in vblank_disable_fn()
176 			vblank_disable_and_save(dev, i);  in vblank_disable_fn()
178 		mtx_unlock(&dev->vbl_lock);  in vblank_disable_fn()
182 void drm_vblank_cleanup(struct drm_device *dev)  in drm_vblank_cleanup()  argument
185 	if (dev->num_crtcs == 0)  in drm_vblank_cleanup()
188 	callout_stop(&dev->vblank_disable_callout);  in drm_vblank_cleanup()
190 	vblank_disable_fn(dev);  in drm_vblank_cleanup()
192 	free(dev->_vblank_count, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
193 	free(dev->vblank_refcount, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
194 	free(dev->vblank_enabled, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
195 	free(dev->last_vblank, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
196 	free(dev->last_vblank_wait, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
197 	free(dev->vblank_inmodeset, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
198 	free(dev->_vblank_time, DRM_MEM_VBLANK);  in drm_vblank_cleanup()
200 	mtx_destroy(&dev->vbl_lock);  in drm_vblank_cleanup()
201 	mtx_destroy(&dev->vblank_time_lock);  in drm_vblank_cleanup()
203 	dev->num_crtcs = 0;  in drm_vblank_cleanup()
207 int drm_vblank_init(struct drm_device *dev, int num_crtcs)  in drm_vblank_init()  argument
211 	callout_init(&dev->vblank_disable_callout, 1);  in drm_vblank_init()
212 	mtx_init(&dev->vbl_lock, "drmvbl", NULL, MTX_DEF);  in drm_vblank_init()
213 	mtx_init(&dev->vblank_time_lock, "drmvtl", NULL, MTX_DEF);  in drm_vblank_init()
215 	dev->num_crtcs = num_crtcs;  in drm_vblank_init()
217 	dev->_vblank_count = malloc(sizeof(atomic_t) * num_crtcs,  in drm_vblank_init()
219 	if (!dev->_vblank_count)  in drm_vblank_init()
222 	dev->vblank_refcount = malloc(sizeof(atomic_t) * num_crtcs,  in drm_vblank_init()
224 	if (!dev->vblank_refcount)  in drm_vblank_init()
227 	dev->vblank_enabled = malloc(num_crtcs * sizeof(int),  in drm_vblank_init()
229 	if (!dev->vblank_enabled)  in drm_vblank_init()
232 	dev->last_vblank = malloc(num_crtcs * sizeof(u32),  in drm_vblank_init()
234 	if (!dev->last_vblank)  in drm_vblank_init()
237 	dev->last_vblank_wait = malloc(num_crtcs * sizeof(u32),  in drm_vblank_init()
239 	if (!dev->last_vblank_wait)  in drm_vblank_init()
242 	dev->vblank_inmodeset = malloc(num_crtcs * sizeof(int),  in drm_vblank_init()
244 	if (!dev->vblank_inmodeset)  in drm_vblank_init()
247 	dev->_vblank_time = malloc(num_crtcs * DRM_VBLANKTIME_RBSIZE *  in drm_vblank_init()
249 	if (!dev->_vblank_time)  in drm_vblank_init()
255 	if (dev->driver->get_vblank_timestamp)  in drm_vblank_init()
262 		atomic_set(&dev->_vblank_count[i], 0);  in drm_vblank_init()
263 		atomic_set(&dev->vblank_refcount[i], 0);  in drm_vblank_init()
266 	dev->vblank_disable_allowed = 0;  in drm_vblank_init()
270 	drm_vblank_cleanup(dev);  in drm_vblank_init()
284 int drm_irq_install(struct drm_device *dev)  in drm_irq_install()  argument
289 	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))  in drm_irq_install()
292 	if (drm_dev_to_irq(dev) == 0)  in drm_irq_install()
295 	DRM_LOCK(dev);  in drm_irq_install()
298 	if (!dev->dev_private) {  in drm_irq_install()
299 		DRM_UNLOCK(dev);  in drm_irq_install()
303 	if (dev->irq_enabled) {  in drm_irq_install()
304 		DRM_UNLOCK(dev);  in drm_irq_install()
307 	dev->irq_enabled = 1;  in drm_irq_install()
308 	DRM_UNLOCK(dev);  in drm_irq_install()
310 	DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));  in drm_irq_install()
313 	if (dev->driver->irq_preinstall)  in drm_irq_install()
314 		dev->driver->irq_preinstall(dev);  in drm_irq_install()
318 	if (!drm_core_check_feature(dev, DRIVER_IRQ_SHARED))  in drm_irq_install()
329 	ret = -bus_setup_intr(dev->dev, dev->irqr, sh_flags, NULL,  in drm_irq_install()
330 	    dev->driver->irq_handler, dev, &dev->irqh);  in drm_irq_install()
333 		device_printf(dev->dev, "Error setting interrupt: %d\n", -ret);  in drm_irq_install()
334 		DRM_LOCK(dev);  in drm_irq_install()
335 		dev->irq_enabled = 0;  in drm_irq_install()
336 		DRM_UNLOCK(dev);  in drm_irq_install()
341 	if (dev->driver->irq_postinstall)  in drm_irq_install()
342 		ret = dev->driver->irq_postinstall(dev);  in drm_irq_install()
345 		DRM_LOCK(dev);  in drm_irq_install()
346 		dev->irq_enabled = 0;  in drm_irq_install()
347 		DRM_UNLOCK(dev);  in drm_irq_install()
348 		bus_teardown_intr(dev->dev, dev->irqr, dev->irqh);  in drm_irq_install()
349 		dev->driver->bus->free_irq(dev);  in drm_irq_install()
363 int drm_irq_uninstall(struct drm_device *dev)  in drm_irq_uninstall()  argument
367 	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))  in drm_irq_uninstall()
370 	DRM_LOCK(dev);  in drm_irq_uninstall()
371 	irq_enabled = dev->irq_enabled;  in drm_irq_uninstall()
372 	dev->irq_enabled = 0;  in drm_irq_uninstall()
373 	DRM_UNLOCK(dev);  in drm_irq_uninstall()
378 	if (dev->num_crtcs) {  in drm_irq_uninstall()
379 		mtx_lock(&dev->vbl_lock);  in drm_irq_uninstall()
380 		for (i = 0; i < dev->num_crtcs; i++) {  in drm_irq_uninstall()
381 			DRM_WAKEUP(&dev->_vblank_count[i]);  in drm_irq_uninstall()
382 			dev->vblank_enabled[i] = 0;  in drm_irq_uninstall()
383 			dev->last_vblank[i] =  in drm_irq_uninstall()
384 				dev->driver->get_vblank_counter(dev, i);  in drm_irq_uninstall()
386 		mtx_unlock(&dev->vbl_lock);  in drm_irq_uninstall()
392 	DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));  in drm_irq_uninstall()
394 	if (dev->driver->irq_uninstall)  in drm_irq_uninstall()
395 		dev->driver->irq_uninstall(dev);  in drm_irq_uninstall()
397 	bus_teardown_intr(dev->dev, dev->irqr, dev->irqh);  in drm_irq_uninstall()
398 	dev->driver->bus->free_irq(dev);  in drm_irq_uninstall()
415 int drm_control(struct drm_device *dev, void *data,  in drm_control()  argument
427 		if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))  in drm_control()
429 		if (drm_core_check_feature(dev, DRIVER_MODESET))  in drm_control()
431 		if (dev->if_version < DRM_IF_VERSION(1, 2) &&  in drm_control()
432 		    ctl->irq != drm_dev_to_irq(dev))  in drm_control()
434 		return drm_irq_install(dev);  in drm_control()
436 		if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))  in drm_control()
438 		if (drm_core_check_feature(dev, DRIVER_MODESET))  in drm_control()
440 		return drm_irq_uninstall(dev);  in drm_control()
543 int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,  in drm_calc_vbltimestamp_from_scanoutpos()  argument
556 	if (crtc < 0 || crtc >= dev->num_crtcs) {  in drm_calc_vbltimestamp_from_scanoutpos()
562 	if (!dev->driver->get_scanout_position) {  in drm_calc_vbltimestamp_from_scanoutpos()
601 		vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos);  in drm_calc_vbltimestamp_from_scanoutpos()
724 u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,  in drm_get_last_vbltimestamp()  argument
733 	if (dev->driver->get_vblank_timestamp && (max_error > 0)) {  in drm_get_last_vbltimestamp()
734 		ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,  in drm_get_last_vbltimestamp()
758 u32 drm_vblank_count(struct drm_device *dev, int crtc)  in drm_vblank_count()  argument
760 	return atomic_read(&dev->_vblank_count[crtc]);  in drm_vblank_count()
778 u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,  in drm_vblank_count_and_time()  argument
789 		cur_vblank = atomic_read(&dev->_vblank_count[crtc]);  in drm_vblank_count_and_time()
790 		*vblanktime = vblanktimestamp(dev, crtc, cur_vblank);  in drm_vblank_count_and_time()
792 	} while (cur_vblank != atomic_read(&dev->_vblank_count[crtc]));  in drm_vblank_count_and_time()
798 static void send_vblank_event(struct drm_device *dev,  in send_vblank_event()  argument
802 	WARN_ON_SMP(!mtx_owned(&dev->event_lock));  in send_vblank_event()
823 void drm_send_vblank_event(struct drm_device *dev, int crtc,  in drm_send_vblank_event()  argument
829 		seq = drm_vblank_count_and_time(dev, crtc, &now);  in drm_send_vblank_event()
835 	send_vblank_event(dev, e, seq, &now);  in drm_send_vblank_event()
855 static void drm_update_vblank_count(struct drm_device *dev, int crtc)  in drm_update_vblank_count()  argument
873 		cur_vblank = dev->driver->get_vblank_counter(dev, crtc);  in drm_update_vblank_count()
874 		rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);  in drm_update_vblank_count()
875 	} while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));  in drm_update_vblank_count()
878 	diff = cur_vblank - dev->last_vblank[crtc];  in drm_update_vblank_count()
879 	if (cur_vblank < dev->last_vblank[crtc]) {  in drm_update_vblank_count()
880 		diff += dev->max_vblank_count;  in drm_update_vblank_count()
883 			  crtc, dev->last_vblank[crtc], cur_vblank, diff);  in drm_update_vblank_count()
894 		tslot = atomic_read(&dev->_vblank_count[crtc]) + diff;  in drm_update_vblank_count()
895 		vblanktimestamp(dev, crtc, tslot) = t_vblank;  in drm_update_vblank_count()
899 	atomic_add(diff, &dev->_vblank_count[crtc]);  in drm_update_vblank_count()
914 int drm_vblank_get(struct drm_device *dev, int crtc)  in drm_vblank_get()  argument
918 	mtx_lock(&dev->vbl_lock);  in drm_vblank_get()
920 	if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) {  in drm_vblank_get()
921 		mtx_lock(&dev->vblank_time_lock);  in drm_vblank_get()
922 		if (!dev->vblank_enabled[crtc]) {  in drm_vblank_get()
929 			ret = dev->driver->enable_vblank(dev, crtc);  in drm_vblank_get()
933 				atomic_dec(&dev->vblank_refcount[crtc]);  in drm_vblank_get()
935 				dev->vblank_enabled[crtc] = 1;  in drm_vblank_get()
936 				drm_update_vblank_count(dev, crtc);  in drm_vblank_get()
939 		mtx_unlock(&dev->vblank_time_lock);  in drm_vblank_get()
941 		if (!dev->vblank_enabled[crtc]) {  in drm_vblank_get()
942 			atomic_dec(&dev->vblank_refcount[crtc]);  in drm_vblank_get()
946 	mtx_unlock(&dev->vbl_lock);  in drm_vblank_get()
960 void drm_vblank_put(struct drm_device *dev, int crtc)  in drm_vblank_put()  argument
962 	BUG_ON(atomic_read(&dev->vblank_refcount[crtc]) == 0);  in drm_vblank_put()
965 	if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) &&  in drm_vblank_put()
967 		callout_reset(&dev->vblank_disable_callout,  in drm_vblank_put()
969 		    vblank_disable_fn, dev);  in drm_vblank_put()
980 void drm_vblank_off(struct drm_device *dev, int crtc)  in drm_vblank_off()  argument
986 	mtx_lock(&dev->vbl_lock);  in drm_vblank_off()
987 	vblank_disable_and_save(dev, crtc);  in drm_vblank_off()
988 	DRM_WAKEUP(&dev->_vblank_count[crtc]);  in drm_vblank_off()
991 	seq = drm_vblank_count_and_time(dev, crtc, &now);  in drm_vblank_off()
993 	mtx_lock(&dev->event_lock);  in drm_vblank_off()
994 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {  in drm_vblank_off()
1001 		drm_vblank_put(dev, e->pipe);  in drm_vblank_off()
1002 		send_vblank_event(dev, e, seq, &now);  in drm_vblank_off()
1004 	mtx_unlock(&dev->event_lock);  in drm_vblank_off()
1006 	mtx_unlock(&dev->vbl_lock);  in drm_vblank_off()
1018 void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)  in drm_vblank_pre_modeset()  argument
1021 	if (!dev->num_crtcs)  in drm_vblank_pre_modeset()
1030 	if (!dev->vblank_inmodeset[crtc]) {  in drm_vblank_pre_modeset()
1031 		dev->vblank_inmodeset[crtc] = 0x1;  in drm_vblank_pre_modeset()
1032 		if (drm_vblank_get(dev, crtc) == 0)  in drm_vblank_pre_modeset()
1033 			dev->vblank_inmodeset[crtc] |= 0x2;  in drm_vblank_pre_modeset()
1038 void drm_vblank_post_modeset(struct drm_device *dev, int crtc)  in drm_vblank_post_modeset()  argument
1041 	if (!dev->num_crtcs)  in drm_vblank_post_modeset()
1044 	if (dev->vblank_inmodeset[crtc]) {  in drm_vblank_post_modeset()
1045 		mtx_lock(&dev->vbl_lock);  in drm_vblank_post_modeset()
1046 		dev->vblank_disable_allowed = 1;  in drm_vblank_post_modeset()
1047 		mtx_unlock(&dev->vbl_lock);  in drm_vblank_post_modeset()
1049 		if (dev->vblank_inmodeset[crtc] & 0x2)  in drm_vblank_post_modeset()
1050 			drm_vblank_put(dev, crtc);  in drm_vblank_post_modeset()
1052 		dev->vblank_inmodeset[crtc] = 0;  in drm_vblank_post_modeset()
1068 int drm_modeset_ctl(struct drm_device *dev, void *data,  in drm_modeset_ctl()  argument
1075 	if (!dev->num_crtcs)  in drm_modeset_ctl()
1079 	if (drm_core_check_feature(dev, DRIVER_MODESET))  in drm_modeset_ctl()
1083 	if (crtc >= dev->num_crtcs)  in drm_modeset_ctl()
1088 		drm_vblank_pre_modeset(dev, crtc);  in drm_modeset_ctl()
1091 		drm_vblank_post_modeset(dev, crtc);  in drm_modeset_ctl()
1107 static int drm_queue_vblank_event(struct drm_device *dev, int pipe,  in drm_queue_vblank_event()  argument
1131 	mtx_lock(&dev->event_lock);  in drm_queue_vblank_event()
1139 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_queue_vblank_event()
1155 		drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1156 		send_vblank_event(dev, e, seq, &now);  in drm_queue_vblank_event()
1160 		list_add_tail(&e->base.link, &dev->vblank_event_list);  in drm_queue_vblank_event()
1164 	mtx_unlock(&dev->event_lock);  in drm_queue_vblank_event()
1169 	mtx_unlock(&dev->event_lock);  in drm_queue_vblank_event()
1172 	drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1190 int drm_wait_vblank(struct drm_device *dev, void *data,  in drm_wait_vblank()  argument
1197 	if (/*(!drm_dev_to_irq(dev)) || */(!dev->irq_enabled))  in drm_wait_vblank()
1219 	if (crtc >= dev->num_crtcs)  in drm_wait_vblank()
1222 	ret = drm_vblank_get(dev, crtc);  in drm_wait_vblank()
1227 	seq = drm_vblank_count(dev, crtc);  in drm_wait_vblank()
1244 		return drm_queue_vblank_event(dev, crtc, vblwait, file_priv);  in drm_wait_vblank()
1254 	dev->last_vblank_wait[crtc] = vblwait->request.sequence;  in drm_wait_vblank()
1255 	mtx_lock(&dev->vblank_time_lock);  in drm_wait_vblank()
1256 	while (((drm_vblank_count(dev, crtc) - vblwait->request.sequence) >  in drm_wait_vblank()
1257 	    (1 << 23)) && dev->irq_enabled) {  in drm_wait_vblank()
1266 		ret = -msleep(&dev->_vblank_count[crtc], &dev->vblank_time_lock,  in drm_wait_vblank()
1273 	mtx_unlock(&dev->vblank_time_lock);  in drm_wait_vblank()
1278 		reply_seq = drm_vblank_count_and_time(dev, crtc, &now);  in drm_wait_vblank()
1298 	drm_vblank_put(dev, crtc);  in drm_wait_vblank()
1302 static void drm_handle_vblank_events(struct drm_device *dev, int crtc)  in drm_handle_vblank_events()  argument
1308 	seq = drm_vblank_count_and_time(dev, crtc, &now);  in drm_handle_vblank_events()
1310 	mtx_lock(&dev->event_lock);  in drm_handle_vblank_events()
1312 	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {  in drm_handle_vblank_events()
1322 		drm_vblank_put(dev, e->pipe);  in drm_handle_vblank_events()
1323 		send_vblank_event(dev, e, seq, &now);  in drm_handle_vblank_events()
1326 	mtx_unlock(&dev->event_lock);  in drm_handle_vblank_events()
1339 bool drm_handle_vblank(struct drm_device *dev, int crtc)  in drm_handle_vblank()  argument
1345 	if (!dev->num_crtcs)  in drm_handle_vblank()
1352 	mtx_lock(&dev->vblank_time_lock);  in drm_handle_vblank()
1355 	if (!dev->vblank_enabled[crtc]) {  in drm_handle_vblank()
1356 		mtx_unlock(&dev->vblank_time_lock);  in drm_handle_vblank()
1365 	vblcount = atomic_read(&dev->_vblank_count[crtc]);  in drm_handle_vblank()
1366 	drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);  in drm_handle_vblank()
1370 		  timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));  in drm_handle_vblank()
1383 		vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;  in drm_handle_vblank()
1389 		atomic_inc(&dev->_vblank_count[crtc]);  in drm_handle_vblank()
1396 	DRM_WAKEUP(&dev->_vblank_count[crtc]);  in drm_handle_vblank()
1397 	drm_handle_vblank_events(dev, crtc);  in drm_handle_vblank()
1399 	mtx_unlock(&dev->vblank_time_lock);  in drm_handle_vblank()