1 /*- 2 * Copyright 2003 Eric Anholt 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * ERIC ANHOLT BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 20 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <anholt@FreeBSD.org> 25 * 26 * $FreeBSD: src/sys/dev/drm2/drm_irq.c,v 1.1 2012/05/22 11:07:44 kib Exp $ 27 */ 28 29 /** @file drm_irq.c 30 * Support code for handling setup/teardown of interrupt handlers and 31 * handing interrupt handlers off to the drivers. 32 */ 33 34 #include <drm/drmP.h> 35 36 MALLOC_DEFINE(DRM_MEM_VBLANK, "drm_vblank", "DRM VBLANK Handling Data"); 37 38 /* Access macro for slots in vblank timestamp ringbuffer. */ 39 #define vblanktimestamp(dev, crtc, count) ( \ 40 (dev)->_vblank_time[(crtc) * DRM_VBLANKTIME_RBSIZE + \ 41 ((count) % DRM_VBLANKTIME_RBSIZE)]) 42 43 /* Retry timestamp calculation up to 3 times to satisfy 44 * drm_timestamp_precision before giving up. 45 */ 46 #define DRM_TIMESTAMP_MAXRETRIES 3 47 48 /* Threshold in nanoseconds for detection of redundant 49 * vblank irq in drm_handle_vblank(). 1 msec should be ok. 50 */ 51 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 52 53 int drm_irq_by_busid(struct drm_device *dev, void *data, 54 struct drm_file *file_priv) 55 { 56 struct drm_irq_busid *irq = data; 57 58 if ((irq->busnum >> 8) != dev->pci_domain || 59 (irq->busnum & 0xff) != dev->pci_bus || 60 irq->devnum != dev->pci_slot || 61 irq->funcnum != dev->pci_func) 62 return EINVAL; 63 64 irq->irq = dev->irq; 65 66 DRM_DEBUG("%d:%d:%d => IRQ %d\n", 67 irq->busnum, irq->devnum, irq->funcnum, irq->irq); 68 69 return 0; 70 } 71 72 int 73 drm_irq_install(struct drm_device *dev) 74 { 75 int retcode; 76 77 if (dev->irq == 0 || dev->dev_private == NULL) 78 return (EINVAL); 79 80 DRM_DEBUG("irq=%d\n", dev->irq); 81 82 DRM_LOCK(dev); 83 if (dev->irq_enabled) { 84 DRM_UNLOCK(dev); 85 return EBUSY; 86 } 87 dev->irq_enabled = 1; 88 89 dev->context_flag = 0; 90 91 /* Before installing handler */ 92 if (dev->driver->irq_preinstall) 93 dev->driver->irq_preinstall(dev); 94 DRM_UNLOCK(dev); 95 96 /* Install handler */ 97 retcode = bus_setup_intr(dev->dev, dev->irqr, INTR_MPSAFE, 98 dev->driver->irq_handler, dev, &dev->irqh, &dev->irq_lock); 99 if (retcode != 0) 100 goto err; 101 102 /* After installing handler */ 103 DRM_LOCK(dev); 104 if (dev->driver->irq_postinstall) 105 dev->driver->irq_postinstall(dev); 106 DRM_UNLOCK(dev); 107 108 return (0); 109 err: 110 device_printf(dev->dev, "Error setting interrupt: %d\n", retcode); 111 dev->irq_enabled = 0; 112 113 return (retcode); 114 } 115 116 int drm_irq_uninstall(struct drm_device *dev) 117 { 118 int i; 119 120 if (!dev->irq_enabled) 121 return EINVAL; 122 123 dev->irq_enabled = 0; 124 125 /* 126 * Wake up any waiters so they don't hang. 127 */ 128 if (dev->num_crtcs) { 129 lockmgr(&dev->vbl_lock, LK_EXCLUSIVE); 130 for (i = 0; i < dev->num_crtcs; i++) { 131 wakeup(&dev->_vblank_count[i]); 132 dev->vblank_enabled[i] = 0; 133 dev->last_vblank[i] = 134 dev->driver->get_vblank_counter(dev, i); 135 } 136 lockmgr(&dev->vbl_lock, LK_RELEASE); 137 } 138 139 DRM_DEBUG("irq=%d\n", dev->irq); 140 141 if (dev->driver->irq_uninstall) 142 dev->driver->irq_uninstall(dev); 143 144 DRM_UNLOCK(dev); 145 bus_teardown_intr(dev->dev, dev->irqr, dev->irqh); 146 DRM_LOCK(dev); 147 148 return 0; 149 } 150 151 int drm_control(struct drm_device *dev, void *data, struct drm_file *file_priv) 152 { 153 struct drm_control *ctl = data; 154 int err; 155 156 switch (ctl->func) { 157 case DRM_INST_HANDLER: 158 /* Handle drivers whose DRM used to require IRQ setup but the 159 * no longer does. 160 */ 161 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) 162 return 0; 163 if (drm_core_check_feature(dev, DRIVER_MODESET)) 164 return 0; 165 if (dev->if_version < DRM_IF_VERSION(1, 2) && 166 ctl->irq != dev->irq) 167 return EINVAL; 168 return drm_irq_install(dev); 169 case DRM_UNINST_HANDLER: 170 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) 171 return 0; 172 if (drm_core_check_feature(dev, DRIVER_MODESET)) 173 return 0; 174 DRM_LOCK(dev); 175 err = drm_irq_uninstall(dev); 176 DRM_UNLOCK(dev); 177 return err; 178 default: 179 return EINVAL; 180 } 181 } 182 183 #define NSEC_PER_USEC 1000L 184 #define NSEC_PER_SEC 1000000000L 185 186 int64_t 187 timeval_to_ns(const struct timeval *tv) 188 { 189 return ((int64_t)tv->tv_sec * NSEC_PER_SEC) + 190 tv->tv_usec * NSEC_PER_USEC; 191 } 192 193 struct timeval 194 ns_to_timeval(const int64_t nsec) 195 { 196 struct timeval tv; 197 long rem; 198 199 if (nsec == 0) { 200 tv.tv_sec = 0; 201 tv.tv_usec = 0; 202 return (tv); 203 } 204 205 tv.tv_sec = nsec / NSEC_PER_SEC; 206 rem = nsec % NSEC_PER_SEC; 207 if (rem < 0) { 208 tv.tv_sec--; 209 rem += NSEC_PER_SEC; 210 } 211 tv.tv_usec = rem / 1000; 212 return (tv); 213 } 214 215 /* 216 * Clear vblank timestamp buffer for a crtc. 217 */ 218 static void clear_vblank_timestamps(struct drm_device *dev, int crtc) 219 { 220 memset(&dev->_vblank_time[crtc * DRM_VBLANKTIME_RBSIZE], 0, 221 DRM_VBLANKTIME_RBSIZE * sizeof(struct timeval)); 222 } 223 224 static int64_t 225 abs64(int64_t x) 226 { 227 228 return (x < 0 ? -x : x); 229 } 230 231 /* 232 * Disable vblank irq's on crtc, make sure that last vblank count 233 * of hardware and corresponding consistent software vblank counter 234 * are preserved, even if there are any spurious vblank irq's after 235 * disable. 236 */ 237 static void vblank_disable_and_save(struct drm_device *dev, int crtc) 238 { 239 u32 vblcount; 240 int64_t diff_ns; 241 int vblrc; 242 struct timeval tvblank; 243 244 /* Prevent vblank irq processing while disabling vblank irqs, 245 * so no updates of timestamps or count can happen after we've 246 * disabled. Needed to prevent races in case of delayed irq's. 247 */ 248 lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE); 249 250 dev->driver->disable_vblank(dev, crtc); 251 dev->vblank_enabled[crtc] = 0; 252 253 /* No further vblank irq's will be processed after 254 * this point. Get current hardware vblank count and 255 * vblank timestamp, repeat until they are consistent. 256 * 257 * FIXME: There is still a race condition here and in 258 * drm_update_vblank_count() which can cause off-by-one 259 * reinitialization of software vblank counter. If gpu 260 * vblank counter doesn't increment exactly at the leading 261 * edge of a vblank interval, then we can lose 1 count if 262 * we happen to execute between start of vblank and the 263 * delayed gpu counter increment. 264 */ 265 do { 266 dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc); 267 vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0); 268 } while (dev->last_vblank[crtc] != dev->driver->get_vblank_counter(dev, crtc)); 269 270 /* Compute time difference to stored timestamp of last vblank 271 * as updated by last invocation of drm_handle_vblank() in vblank irq. 272 */ 273 vblcount = atomic_read(&dev->_vblank_count[crtc]); 274 diff_ns = timeval_to_ns(&tvblank) - 275 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount)); 276 277 /* If there is at least 1 msec difference between the last stored 278 * timestamp and tvblank, then we are currently executing our 279 * disable inside a new vblank interval, the tvblank timestamp 280 * corresponds to this new vblank interval and the irq handler 281 * for this vblank didn't run yet and won't run due to our disable. 282 * Therefore we need to do the job of drm_handle_vblank() and 283 * increment the vblank counter by one to account for this vblank. 284 * 285 * Skip this step if there isn't any high precision timestamp 286 * available. In that case we can't account for this and just 287 * hope for the best. 288 */ 289 if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) { 290 atomic_inc(&dev->_vblank_count[crtc]); 291 } 292 293 /* Invalidate all timestamps while vblank irq's are off. */ 294 clear_vblank_timestamps(dev, crtc); 295 296 lockmgr(&dev->vblank_time_lock, LK_RELEASE); 297 } 298 299 static void vblank_disable_fn(void * arg) 300 { 301 struct drm_device *dev = (struct drm_device *)arg; 302 int i; 303 304 if (!dev->vblank_disable_allowed) 305 return; 306 307 for (i = 0; i < dev->num_crtcs; i++) { 308 lockmgr(&dev->vbl_lock, LK_EXCLUSIVE); 309 if (atomic_read(&dev->vblank_refcount[i]) == 0 && 310 dev->vblank_enabled[i]) { 311 DRM_DEBUG("disabling vblank on crtc %d\n", i); 312 vblank_disable_and_save(dev, i); 313 } 314 lockmgr(&dev->vbl_lock, LK_RELEASE); 315 } 316 } 317 318 void drm_vblank_cleanup(struct drm_device *dev) 319 { 320 /* Bail if the driver didn't call drm_vblank_init() */ 321 if (dev->num_crtcs == 0) 322 return; 323 324 callout_stop(&dev->vblank_disable_callout); 325 326 vblank_disable_fn(dev); 327 328 drm_free(dev->_vblank_count, DRM_MEM_VBLANK); 329 drm_free(dev->vblank_refcount, DRM_MEM_VBLANK); 330 drm_free(dev->vblank_enabled, DRM_MEM_VBLANK); 331 drm_free(dev->last_vblank, DRM_MEM_VBLANK); 332 drm_free(dev->last_vblank_wait, DRM_MEM_VBLANK); 333 drm_free(dev->vblank_inmodeset, DRM_MEM_VBLANK); 334 drm_free(dev->_vblank_time, DRM_MEM_VBLANK); 335 336 dev->num_crtcs = 0; 337 } 338 339 int drm_vblank_init(struct drm_device *dev, int num_crtcs) 340 { 341 int i; 342 343 callout_init_mp(&dev->vblank_disable_callout); 344 #if 0 345 mtx_init(&dev->vbl_lock, "drmvbl", NULL, MTX_DEF); 346 #endif 347 lockinit(&dev->vblank_time_lock, "drmvtl", 0, LK_CANRECURSE); 348 349 dev->num_crtcs = num_crtcs; 350 351 dev->_vblank_count = kmalloc(sizeof(atomic_t) * num_crtcs, 352 DRM_MEM_VBLANK, M_WAITOK); 353 dev->vblank_refcount = kmalloc(sizeof(atomic_t) * num_crtcs, 354 DRM_MEM_VBLANK, M_WAITOK); 355 dev->vblank_enabled = kmalloc(num_crtcs * sizeof(int), 356 DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 357 dev->last_vblank = kmalloc(num_crtcs * sizeof(u32), 358 DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 359 dev->last_vblank_wait = kmalloc(num_crtcs * sizeof(u32), 360 DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 361 dev->vblank_inmodeset = kmalloc(num_crtcs * sizeof(int), 362 DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 363 dev->_vblank_time = kmalloc(num_crtcs * DRM_VBLANKTIME_RBSIZE * 364 sizeof(struct timeval), DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 365 DRM_INFO("Supports vblank timestamp caching Rev 1 (10.10.2010).\n"); 366 367 /* Driver specific high-precision vblank timestamping supported? */ 368 if (dev->driver->get_vblank_timestamp) 369 DRM_INFO("Driver supports precise vblank timestamp query.\n"); 370 else 371 DRM_INFO("No driver support for vblank timestamp query.\n"); 372 373 /* Zero per-crtc vblank stuff */ 374 for (i = 0; i < num_crtcs; i++) { 375 atomic_set(&dev->_vblank_count[i], 0); 376 atomic_set(&dev->vblank_refcount[i], 0); 377 } 378 379 dev->vblank_disable_allowed = 0; 380 return 0; 381 } 382 383 void 384 drm_calc_timestamping_constants(struct drm_crtc *crtc) 385 { 386 int64_t linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0; 387 uint64_t dotclock; 388 389 /* Dot clock in Hz: */ 390 dotclock = (uint64_t) crtc->hwmode.clock * 1000; 391 392 /* Fields of interlaced scanout modes are only halve a frame duration. 393 * Double the dotclock to get halve the frame-/line-/pixelduration. 394 */ 395 if (crtc->hwmode.flags & DRM_MODE_FLAG_INTERLACE) 396 dotclock *= 2; 397 398 /* Valid dotclock? */ 399 if (dotclock > 0) { 400 /* Convert scanline length in pixels and video dot clock to 401 * line duration, frame duration and pixel duration in 402 * nanoseconds: 403 */ 404 pixeldur_ns = (int64_t)1000000000 / dotclock; 405 linedur_ns = ((uint64_t)crtc->hwmode.crtc_htotal * 406 1000000000) / dotclock; 407 framedur_ns = (int64_t)crtc->hwmode.crtc_vtotal * linedur_ns; 408 } else 409 DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n", 410 crtc->base.id); 411 412 crtc->pixeldur_ns = pixeldur_ns; 413 crtc->linedur_ns = linedur_ns; 414 crtc->framedur_ns = framedur_ns; 415 416 DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n", 417 crtc->base.id, crtc->hwmode.crtc_htotal, 418 crtc->hwmode.crtc_vtotal, crtc->hwmode.crtc_vdisplay); 419 DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n", 420 crtc->base.id, (int) dotclock/1000, (int) framedur_ns, 421 (int) linedur_ns, (int) pixeldur_ns); 422 } 423 424 /** 425 * drm_calc_vbltimestamp_from_scanoutpos - helper routine for kms 426 * drivers. Implements calculation of exact vblank timestamps from 427 * given drm_display_mode timings and current video scanout position 428 * of a crtc. This can be called from within get_vblank_timestamp() 429 * implementation of a kms driver to implement the actual timestamping. 430 * 431 * Should return timestamps conforming to the OML_sync_control OpenML 432 * extension specification. The timestamp corresponds to the end of 433 * the vblank interval, aka start of scanout of topmost-leftmost display 434 * pixel in the following video frame. 435 * 436 * Requires support for optional dev->driver->get_scanout_position() 437 * in kms driver, plus a bit of setup code to provide a drm_display_mode 438 * that corresponds to the true scanout timing. 439 * 440 * The current implementation only handles standard video modes. It 441 * returns as no operation if a doublescan or interlaced video mode is 442 * active. Higher level code is expected to handle this. 443 * 444 * @dev: DRM device. 445 * @crtc: Which crtc's vblank timestamp to retrieve. 446 * @max_error: Desired maximum allowable error in timestamps (nanosecs). 447 * On return contains true maximum error of timestamp. 448 * @vblank_time: Pointer to struct timeval which should receive the timestamp. 449 * @flags: Flags to pass to driver: 450 * 0 = Default. 451 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler. 452 * @refcrtc: drm_crtc* of crtc which defines scanout timing. 453 * 454 * Returns negative value on error, failure or if not supported in current 455 * video mode: 456 * 457 * -EINVAL - Invalid crtc. 458 * -EAGAIN - Temporary unavailable, e.g., called before initial modeset. 459 * -ENOTSUPP - Function not supported in current display mode. 460 * -EIO - Failed, e.g., due to failed scanout position query. 461 * 462 * Returns or'ed positive status flags on success: 463 * 464 * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping. 465 * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval. 466 * 467 */ 468 int 469 drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, 470 int *max_error, struct timeval *vblank_time, unsigned flags, 471 struct drm_crtc *refcrtc) 472 { 473 struct timeval stime, raw_time; 474 struct drm_display_mode *mode; 475 int vbl_status, vtotal, vdisplay; 476 int vpos, hpos, i; 477 int64_t framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns; 478 bool invbl; 479 480 if (crtc < 0 || crtc >= dev->num_crtcs) { 481 DRM_ERROR("Invalid crtc %d\n", crtc); 482 return -EINVAL; 483 } 484 485 /* Scanout position query not supported? Should not happen. */ 486 if (!dev->driver->get_scanout_position) { 487 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n"); 488 return -EIO; 489 } 490 491 mode = &refcrtc->hwmode; 492 vtotal = mode->crtc_vtotal; 493 vdisplay = mode->crtc_vdisplay; 494 495 /* Durations of frames, lines, pixels in nanoseconds. */ 496 framedur_ns = refcrtc->framedur_ns; 497 linedur_ns = refcrtc->linedur_ns; 498 pixeldur_ns = refcrtc->pixeldur_ns; 499 500 /* If mode timing undefined, just return as no-op: 501 * Happens during initial modesetting of a crtc. 502 */ 503 if (vtotal <= 0 || vdisplay <= 0 || framedur_ns == 0) { 504 DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc); 505 return -EAGAIN; 506 } 507 508 /* Get current scanout position with system timestamp. 509 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times 510 * if single query takes longer than max_error nanoseconds. 511 * 512 * This guarantees a tight bound on maximum error if 513 * code gets preempted or delayed for some reason. 514 */ 515 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) { 516 /* Disable preemption to make it very likely to 517 * succeed in the first iteration. 518 */ 519 crit_enter(); 520 521 /* Get system timestamp before query. */ 522 getmicrouptime(&stime); 523 524 /* Get vertical and horizontal scanout pos. vpos, hpos. */ 525 vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos); 526 527 /* Get system timestamp after query. */ 528 getmicrouptime(&raw_time); 529 530 crit_exit(); 531 532 /* Return as no-op if scanout query unsupported or failed. */ 533 if (!(vbl_status & DRM_SCANOUTPOS_VALID)) { 534 DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n", 535 crtc, vbl_status); 536 return -EIO; 537 } 538 539 duration_ns = timeval_to_ns(&raw_time) - timeval_to_ns(&stime); 540 541 /* Accept result with < max_error nsecs timing uncertainty. */ 542 if (duration_ns <= (int64_t) *max_error) 543 break; 544 } 545 546 /* Noisy system timing? */ 547 if (i == DRM_TIMESTAMP_MAXRETRIES) { 548 DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n", 549 crtc, (int) duration_ns/1000, *max_error/1000, i); 550 } 551 552 /* Return upper bound of timestamp precision error. */ 553 *max_error = (int) duration_ns; 554 555 /* Check if in vblank area: 556 * vpos is >=0 in video scanout area, but negative 557 * within vblank area, counting down the number of lines until 558 * start of scanout. 559 */ 560 invbl = vbl_status & DRM_SCANOUTPOS_INVBL; 561 562 /* Convert scanout position into elapsed time at raw_time query 563 * since start of scanout at first display scanline. delta_ns 564 * can be negative if start of scanout hasn't happened yet. 565 */ 566 delta_ns = (int64_t)vpos * linedur_ns + (int64_t)hpos * pixeldur_ns; 567 568 /* Is vpos outside nominal vblank area, but less than 569 * 1/100 of a frame height away from start of vblank? 570 * If so, assume this isn't a massively delayed vblank 571 * interrupt, but a vblank interrupt that fired a few 572 * microseconds before true start of vblank. Compensate 573 * by adding a full frame duration to the final timestamp. 574 * Happens, e.g., on ATI R500, R600. 575 * 576 * We only do this if DRM_CALLED_FROM_VBLIRQ. 577 */ 578 if ((flags & DRM_CALLED_FROM_VBLIRQ) && !invbl && 579 ((vdisplay - vpos) < vtotal / 100)) { 580 delta_ns = delta_ns - framedur_ns; 581 582 /* Signal this correction as "applied". */ 583 vbl_status |= 0x8; 584 } 585 586 /* Subtract time delta from raw timestamp to get final 587 * vblank_time timestamp for end of vblank. 588 */ 589 *vblank_time = ns_to_timeval(timeval_to_ns(&raw_time) - delta_ns); 590 591 DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %jd.%jd -> %jd.%jd [e %d us, %d rep]\n", 592 crtc, (int)vbl_status, hpos, vpos, (uintmax_t)raw_time.tv_sec, 593 (uintmax_t)raw_time.tv_usec, (uintmax_t)vblank_time->tv_sec, 594 (uintmax_t)vblank_time->tv_usec, (int)duration_ns/1000, i); 595 596 vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD; 597 if (invbl) 598 vbl_status |= DRM_VBLANKTIME_INVBL; 599 600 return vbl_status; 601 } 602 603 /** 604 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent 605 * vblank interval. 606 * 607 * @dev: DRM device 608 * @crtc: which crtc's vblank timestamp to retrieve 609 * @tvblank: Pointer to target struct timeval which should receive the timestamp 610 * @flags: Flags to pass to driver: 611 * 0 = Default. 612 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler. 613 * 614 * Fetches the system timestamp corresponding to the time of the most recent 615 * vblank interval on specified crtc. May call into kms-driver to 616 * compute the timestamp with a high-precision GPU specific method. 617 * 618 * Returns zero if timestamp originates from uncorrected do_gettimeofday() 619 * call, i.e., it isn't very precisely locked to the true vblank. 620 * 621 * Returns non-zero if timestamp is considered to be very precise. 622 */ 623 u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, 624 struct timeval *tvblank, unsigned flags) 625 { 626 int ret = 0; 627 628 /* Define requested maximum error on timestamps (nanoseconds). */ 629 int max_error = (int) drm_timestamp_precision * 1000; 630 631 /* Query driver if possible and precision timestamping enabled. */ 632 if (dev->driver->get_vblank_timestamp && (max_error > 0)) { 633 ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error, 634 tvblank, flags); 635 if (ret > 0) 636 return (u32) ret; 637 } 638 639 /* GPU high precision timestamp query unsupported or failed. 640 * Return gettimeofday timestamp as best estimate. 641 */ 642 microtime(tvblank); 643 644 return 0; 645 } 646 647 /** 648 * drm_vblank_count - retrieve "cooked" vblank counter value 649 * @dev: DRM device 650 * @crtc: which counter to retrieve 651 * 652 * Fetches the "cooked" vblank count value that represents the number of 653 * vblank events since the system was booted, including lost events due to 654 * modesetting activity. 655 */ 656 u32 drm_vblank_count(struct drm_device *dev, int crtc) 657 { 658 return atomic_read(&dev->_vblank_count[crtc]); 659 } 660 661 /** 662 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value 663 * and the system timestamp corresponding to that vblank counter value. 664 * 665 * @dev: DRM device 666 * @crtc: which counter to retrieve 667 * @vblanktime: Pointer to struct timeval to receive the vblank timestamp. 668 * 669 * Fetches the "cooked" vblank count value that represents the number of 670 * vblank events since the system was booted, including lost events due to 671 * modesetting activity. Returns corresponding system timestamp of the time 672 * of the vblank interval that corresponds to the current value vblank counter 673 * value. 674 */ 675 u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc, 676 struct timeval *vblanktime) 677 { 678 u32 cur_vblank; 679 680 /* Read timestamp from slot of _vblank_time ringbuffer 681 * that corresponds to current vblank count. Retry if 682 * count has incremented during readout. This works like 683 * a seqlock. 684 */ 685 do { 686 cur_vblank = atomic_read(&dev->_vblank_count[crtc]); 687 *vblanktime = vblanktimestamp(dev, crtc, cur_vblank); 688 cpu_lfence(); 689 } while (cur_vblank != atomic_read(&dev->_vblank_count[crtc])); 690 691 return cur_vblank; 692 } 693 694 /** 695 * drm_update_vblank_count - update the master vblank counter 696 * @dev: DRM device 697 * @crtc: counter to update 698 * 699 * Call back into the driver to update the appropriate vblank counter 700 * (specified by @crtc). Deal with wraparound, if it occurred, and 701 * update the last read value so we can deal with wraparound on the next 702 * call if necessary. 703 * 704 * Only necessary when going from off->on, to account for frames we 705 * didn't get an interrupt for. 706 * 707 * Note: caller must hold dev->vbl_lock since this reads & writes 708 * device vblank fields. 709 */ 710 static void drm_update_vblank_count(struct drm_device *dev, int crtc) 711 { 712 u32 cur_vblank, diff, tslot, rc; 713 struct timeval t_vblank; 714 715 /* 716 * Interrupts were disabled prior to this call, so deal with counter 717 * wrap if needed. 718 * NOTE! It's possible we lost a full dev->max_vblank_count events 719 * here if the register is small or we had vblank interrupts off for 720 * a long time. 721 * 722 * We repeat the hardware vblank counter & timestamp query until 723 * we get consistent results. This to prevent races between gpu 724 * updating its hardware counter while we are retrieving the 725 * corresponding vblank timestamp. 726 */ 727 do { 728 cur_vblank = dev->driver->get_vblank_counter(dev, crtc); 729 rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0); 730 } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc)); 731 732 /* Deal with counter wrap */ 733 diff = cur_vblank - dev->last_vblank[crtc]; 734 if (cur_vblank < dev->last_vblank[crtc]) { 735 diff += dev->max_vblank_count; 736 737 DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n", 738 crtc, dev->last_vblank[crtc], cur_vblank, diff); 739 } 740 741 DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n", 742 crtc, diff); 743 744 /* Reinitialize corresponding vblank timestamp if high-precision query 745 * available. Skip this step if query unsupported or failed. Will 746 * reinitialize delayed at next vblank interrupt in that case. 747 */ 748 if (rc) { 749 tslot = atomic_read(&dev->_vblank_count[crtc]) + diff; 750 vblanktimestamp(dev, crtc, tslot) = t_vblank; 751 } 752 753 atomic_add(diff, &dev->_vblank_count[crtc]); 754 } 755 756 /** 757 * drm_vblank_get - get a reference count on vblank events 758 * @dev: DRM device 759 * @crtc: which CRTC to own 760 * 761 * Acquire a reference count on vblank events to avoid having them disabled 762 * while in use. 763 * 764 * RETURNS 765 * Zero on success, nonzero on failure. 766 */ 767 int drm_vblank_get(struct drm_device *dev, int crtc) 768 { 769 int ret = 0; 770 771 lockmgr(&dev->vbl_lock, LK_EXCLUSIVE); 772 /* Going from 0->1 means we have to enable interrupts again */ 773 if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) { 774 lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE); 775 if (!dev->vblank_enabled[crtc]) { 776 /* Enable vblank irqs under vblank_time_lock protection. 777 * All vblank count & timestamp updates are held off 778 * until we are done reinitializing master counter and 779 * timestamps. Filtercode in drm_handle_vblank() will 780 * prevent double-accounting of same vblank interval. 781 */ 782 ret = -dev->driver->enable_vblank(dev, crtc); 783 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", 784 crtc, ret); 785 if (ret) 786 atomic_dec(&dev->vblank_refcount[crtc]); 787 else { 788 dev->vblank_enabled[crtc] = 1; 789 drm_update_vblank_count(dev, crtc); 790 } 791 } 792 lockmgr(&dev->vblank_time_lock, LK_RELEASE); 793 } else { 794 if (!dev->vblank_enabled[crtc]) { 795 atomic_dec(&dev->vblank_refcount[crtc]); 796 ret = EINVAL; 797 } 798 } 799 lockmgr(&dev->vbl_lock, LK_RELEASE); 800 801 return ret; 802 } 803 804 /** 805 * drm_vblank_put - give up ownership of vblank events 806 * @dev: DRM device 807 * @crtc: which counter to give up 808 * 809 * Release ownership of a given vblank counter, turning off interrupts 810 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds. 811 */ 812 void drm_vblank_put(struct drm_device *dev, int crtc) 813 { 814 KASSERT(atomic_read(&dev->vblank_refcount[crtc]) != 0, 815 ("Too many drm_vblank_put for crtc %d", crtc)); 816 817 /* Last user schedules interrupt disable */ 818 if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) && 819 (drm_vblank_offdelay > 0)) 820 callout_reset(&dev->vblank_disable_callout, 821 (drm_vblank_offdelay * DRM_HZ) / 1000, 822 vblank_disable_fn, dev); 823 } 824 825 void drm_vblank_off(struct drm_device *dev, int crtc) 826 { 827 struct drm_pending_vblank_event *e, *t; 828 struct timeval now; 829 unsigned int seq; 830 831 lockmgr(&dev->vbl_lock, LK_EXCLUSIVE); 832 vblank_disable_and_save(dev, crtc); 833 lockmgr(&dev->event_lock, LK_EXCLUSIVE); 834 wakeup(&dev->_vblank_count[crtc]); 835 836 /* Send any queued vblank events, lest the natives grow disquiet */ 837 seq = drm_vblank_count_and_time(dev, crtc, &now); 838 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 839 if (e->pipe != crtc) 840 continue; 841 DRM_DEBUG("Sending premature vblank event on disable: \ 842 wanted %d, current %d\n", 843 e->event.sequence, seq); 844 845 e->event.sequence = seq; 846 e->event.tv_sec = now.tv_sec; 847 e->event.tv_usec = now.tv_usec; 848 drm_vblank_put(dev, e->pipe); 849 list_move_tail(&e->base.link, &e->base.file_priv->event_list); 850 drm_event_wakeup(&e->base); 851 } 852 853 lockmgr(&dev->event_lock, LK_RELEASE); 854 lockmgr(&dev->vbl_lock, LK_RELEASE); 855 } 856 857 /** 858 * drm_vblank_pre_modeset - account for vblanks across mode sets 859 * @dev: DRM device 860 * @crtc: CRTC in question 861 * @post: post or pre mode set? 862 * 863 * Account for vblank events across mode setting events, which will likely 864 * reset the hardware frame counter. 865 */ 866 void drm_vblank_pre_modeset(struct drm_device *dev, int crtc) 867 { 868 /* vblank is not initialized (IRQ not installed ?) */ 869 if (!dev->num_crtcs) 870 return; 871 /* 872 * To avoid all the problems that might happen if interrupts 873 * were enabled/disabled around or between these calls, we just 874 * have the kernel take a reference on the CRTC (just once though 875 * to avoid corrupting the count if multiple, mismatch calls occur), 876 * so that interrupts remain enabled in the interim. 877 */ 878 if (!dev->vblank_inmodeset[crtc]) { 879 dev->vblank_inmodeset[crtc] = 0x1; 880 if (drm_vblank_get(dev, crtc) == 0) 881 dev->vblank_inmodeset[crtc] |= 0x2; 882 } 883 } 884 885 void drm_vblank_post_modeset(struct drm_device *dev, int crtc) 886 { 887 888 if (dev->vblank_inmodeset[crtc]) { 889 lockmgr(&dev->vbl_lock, LK_EXCLUSIVE); 890 dev->vblank_disable_allowed = 1; 891 lockmgr(&dev->vbl_lock, LK_RELEASE); 892 893 if (dev->vblank_inmodeset[crtc] & 0x2) 894 drm_vblank_put(dev, crtc); 895 896 dev->vblank_inmodeset[crtc] = 0; 897 } 898 } 899 900 /** 901 * drm_modeset_ctl - handle vblank event counter changes across mode switch 902 * @DRM_IOCTL_ARGS: standard ioctl arguments 903 * 904 * Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET 905 * ioctls around modesetting so that any lost vblank events are accounted for. 906 * 907 * Generally the counter will reset across mode sets. If interrupts are 908 * enabled around this call, we don't have to do anything since the counter 909 * will have already been incremented. 910 */ 911 int drm_modeset_ctl(struct drm_device *dev, void *data, 912 struct drm_file *file_priv) 913 { 914 struct drm_modeset_ctl *modeset = data; 915 int ret = 0; 916 unsigned int crtc; 917 918 /* If drm_vblank_init() hasn't been called yet, just no-op */ 919 if (!dev->num_crtcs) 920 goto out; 921 922 crtc = modeset->crtc; 923 if (crtc >= dev->num_crtcs) { 924 ret = -EINVAL; 925 goto out; 926 } 927 928 switch (modeset->cmd) { 929 case _DRM_PRE_MODESET: 930 drm_vblank_pre_modeset(dev, crtc); 931 break; 932 case _DRM_POST_MODESET: 933 drm_vblank_post_modeset(dev, crtc); 934 break; 935 default: 936 ret = -EINVAL; 937 break; 938 } 939 940 out: 941 return ret; 942 } 943 944 static void 945 drm_vblank_event_destroy(struct drm_pending_event *e) 946 { 947 948 drm_free(e, DRM_MEM_VBLANK); 949 } 950 951 static int drm_queue_vblank_event(struct drm_device *dev, int pipe, 952 union drm_wait_vblank *vblwait, 953 struct drm_file *file_priv) 954 { 955 struct drm_pending_vblank_event *e; 956 struct timeval now; 957 unsigned int seq; 958 int ret; 959 960 e = kmalloc(sizeof *e, DRM_MEM_VBLANK, M_WAITOK | M_ZERO); 961 962 e->pipe = pipe; 963 e->base.pid = curproc->p_pid; 964 e->event.base.type = DRM_EVENT_VBLANK; 965 e->event.base.length = sizeof e->event; 966 e->event.user_data = vblwait->request.signal; 967 e->base.event = &e->event.base; 968 e->base.file_priv = file_priv; 969 e->base.destroy = drm_vblank_event_destroy; 970 971 lockmgr(&dev->event_lock, LK_EXCLUSIVE); 972 973 if (file_priv->event_space < sizeof e->event) { 974 ret = EBUSY; 975 goto err_unlock; 976 } 977 978 file_priv->event_space -= sizeof e->event; 979 seq = drm_vblank_count_and_time(dev, pipe, &now); 980 981 if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) && 982 (seq - vblwait->request.sequence) <= (1 << 23)) { 983 vblwait->request.sequence = seq + 1; 984 vblwait->reply.sequence = vblwait->request.sequence; 985 } 986 987 DRM_DEBUG("event on vblank count %d, current %d, crtc %d\n", 988 vblwait->request.sequence, seq, pipe); 989 990 e->event.sequence = vblwait->request.sequence; 991 if ((seq - vblwait->request.sequence) <= (1 << 23)) { 992 e->event.sequence = seq; 993 e->event.tv_sec = now.tv_sec; 994 e->event.tv_usec = now.tv_usec; 995 drm_vblank_put(dev, pipe); 996 list_add_tail(&e->base.link, &e->base.file_priv->event_list); 997 drm_event_wakeup(&e->base); 998 vblwait->reply.sequence = seq; 999 } else { 1000 /* drm_handle_vblank_events will call drm_vblank_put */ 1001 list_add_tail(&e->base.link, &dev->vblank_event_list); 1002 vblwait->reply.sequence = vblwait->request.sequence; 1003 } 1004 1005 lockmgr(&dev->event_lock, LK_RELEASE); 1006 1007 return 0; 1008 1009 err_unlock: 1010 lockmgr(&dev->event_lock, LK_RELEASE); 1011 drm_free(e, DRM_MEM_VBLANK); 1012 drm_vblank_put(dev, pipe); 1013 return ret; 1014 } 1015 1016 /** 1017 * Wait for VBLANK. 1018 * 1019 * \param inode device inode. 1020 * \param file_priv DRM file private. 1021 * \param cmd command. 1022 * \param data user argument, pointing to a drm_wait_vblank structure. 1023 * \return zero on success or a negative number on failure. 1024 * 1025 * This function enables the vblank interrupt on the pipe requested, then 1026 * sleeps waiting for the requested sequence number to occur, and drops 1027 * the vblank interrupt refcount afterwards. (vblank irq disable follows that 1028 * after a timeout with no further vblank waits scheduled). 1029 */ 1030 int drm_wait_vblank(struct drm_device *dev, void *data, 1031 struct drm_file *file_priv) 1032 { 1033 union drm_wait_vblank *vblwait = data; 1034 int ret = 0; 1035 unsigned int flags, seq, crtc, high_crtc; 1036 1037 if (/*(!drm_dev_to_irq(dev)) || */(!dev->irq_enabled)) 1038 return (EINVAL); 1039 1040 if (vblwait->request.type & _DRM_VBLANK_SIGNAL) 1041 return (EINVAL); 1042 1043 if (vblwait->request.type & 1044 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1045 _DRM_VBLANK_HIGH_CRTC_MASK)) { 1046 DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n", 1047 vblwait->request.type, 1048 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1049 _DRM_VBLANK_HIGH_CRTC_MASK)); 1050 return (EINVAL); 1051 } 1052 1053 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; 1054 high_crtc = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); 1055 if (high_crtc) 1056 crtc = high_crtc >> _DRM_VBLANK_HIGH_CRTC_SHIFT; 1057 else 1058 crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; 1059 if (crtc >= dev->num_crtcs) 1060 return (EINVAL); 1061 1062 ret = drm_vblank_get(dev, crtc); 1063 if (ret) { 1064 DRM_DEBUG("failed to acquire vblank counter, %d\n", ret); 1065 return (ret); 1066 } 1067 seq = drm_vblank_count(dev, crtc); 1068 1069 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { 1070 case _DRM_VBLANK_RELATIVE: 1071 vblwait->request.sequence += seq; 1072 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; 1073 case _DRM_VBLANK_ABSOLUTE: 1074 break; 1075 default: 1076 ret = (EINVAL); 1077 goto done; 1078 } 1079 1080 if (flags & _DRM_VBLANK_EVENT) { 1081 /* must hold on to the vblank ref until the event fires 1082 * drm_vblank_put will be called asynchronously 1083 */ 1084 return drm_queue_vblank_event(dev, crtc, vblwait, file_priv); 1085 } 1086 1087 if ((flags & _DRM_VBLANK_NEXTONMISS) && 1088 (seq - vblwait->request.sequence) <= (1<<23)) { 1089 vblwait->request.sequence = seq + 1; 1090 } 1091 1092 dev->last_vblank_wait[crtc] = vblwait->request.sequence; 1093 lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE); 1094 while (((drm_vblank_count(dev, crtc) - vblwait->request.sequence) > 1095 (1 << 23)) && dev->irq_enabled) { 1096 /* 1097 * The wakeups from the drm_irq_uninstall() and 1098 * drm_vblank_off() may be lost there since vbl_lock 1099 * is not held. Then, the timeout will wake us; the 3 1100 * seconds delay should not be a problem for 1101 * application when crtc is disabled or irq 1102 * uninstalled anyway. 1103 */ 1104 ret = lksleep(&dev->_vblank_count[crtc], &dev->vblank_time_lock, 1105 PCATCH, "drmvbl", 3 * hz); 1106 if (ret != 0) 1107 break; 1108 } 1109 lockmgr(&dev->vblank_time_lock, LK_RELEASE); 1110 if (ret != EINTR) { 1111 struct timeval now; 1112 long reply_seq; 1113 1114 reply_seq = drm_vblank_count_and_time(dev, crtc, &now); 1115 vblwait->reply.sequence = reply_seq; 1116 vblwait->reply.tval_sec = now.tv_sec; 1117 vblwait->reply.tval_usec = now.tv_usec; 1118 } 1119 1120 done: 1121 drm_vblank_put(dev, crtc); 1122 return ret; 1123 } 1124 1125 void drm_handle_vblank_events(struct drm_device *dev, int crtc) 1126 { 1127 struct drm_pending_vblank_event *e, *t; 1128 struct timeval now; 1129 unsigned int seq; 1130 1131 seq = drm_vblank_count_and_time(dev, crtc, &now); 1132 1133 lockmgr(&dev->event_lock, LK_EXCLUSIVE); 1134 1135 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1136 if (e->pipe != crtc) 1137 continue; 1138 if ((seq - e->event.sequence) > (1<<23)) 1139 continue; 1140 1141 e->event.sequence = seq; 1142 e->event.tv_sec = now.tv_sec; 1143 e->event.tv_usec = now.tv_usec; 1144 drm_vblank_put(dev, e->pipe); 1145 list_move_tail(&e->base.link, &e->base.file_priv->event_list); 1146 drm_event_wakeup(&e->base); 1147 } 1148 1149 lockmgr(&dev->event_lock, LK_RELEASE); 1150 } 1151 1152 /** 1153 * drm_handle_vblank - handle a vblank event 1154 * @dev: DRM device 1155 * @crtc: where this event occurred 1156 * 1157 * Drivers should call this routine in their vblank interrupt handlers to 1158 * update the vblank counter and send any signals that may be pending. 1159 */ 1160 bool drm_handle_vblank(struct drm_device *dev, int crtc) 1161 { 1162 u32 vblcount; 1163 int64_t diff_ns; 1164 struct timeval tvblank; 1165 1166 if (!dev->num_crtcs) 1167 return false; 1168 1169 /* Need timestamp lock to prevent concurrent execution with 1170 * vblank enable/disable, as this would cause inconsistent 1171 * or corrupted timestamps and vblank counts. 1172 */ 1173 lockmgr(&dev->vblank_time_lock, LK_EXCLUSIVE); 1174 1175 /* Vblank irq handling disabled. Nothing to do. */ 1176 if (!dev->vblank_enabled[crtc]) { 1177 lockmgr(&dev->vblank_time_lock, LK_RELEASE); 1178 return false; 1179 } 1180 1181 /* Fetch corresponding timestamp for this vblank interval from 1182 * driver and store it in proper slot of timestamp ringbuffer. 1183 */ 1184 1185 /* Get current timestamp and count. */ 1186 vblcount = atomic_read(&dev->_vblank_count[crtc]); 1187 drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ); 1188 1189 /* Compute time difference to timestamp of last vblank */ 1190 diff_ns = timeval_to_ns(&tvblank) - 1191 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount)); 1192 1193 /* Update vblank timestamp and count if at least 1194 * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds 1195 * difference between last stored timestamp and current 1196 * timestamp. A smaller difference means basically 1197 * identical timestamps. Happens if this vblank has 1198 * been already processed and this is a redundant call, 1199 * e.g., due to spurious vblank interrupts. We need to 1200 * ignore those for accounting. 1201 */ 1202 if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) { 1203 /* Store new timestamp in ringbuffer. */ 1204 vblanktimestamp(dev, crtc, vblcount + 1) = tvblank; 1205 1206 /* Increment cooked vblank count. This also atomically commits 1207 * the timestamp computed above. 1208 */ 1209 atomic_inc(&dev->_vblank_count[crtc]); 1210 } else { 1211 DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n", 1212 crtc, (int) diff_ns); 1213 } 1214 1215 wakeup(&dev->_vblank_count[crtc]); 1216 drm_handle_vblank_events(dev, crtc); 1217 1218 lockmgr(&dev->vblank_time_lock, LK_RELEASE); 1219 return true; 1220 } 1221