2 * drm_irq.c IRQ and vblank support
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
27 #include <linux/export.h>
28 #include <linux/moduleparam.h>
30 #include <drm/drm_crtc.h>
31 #include <drm/drm_drv.h>
32 #include <drm/drm_framebuffer.h>
33 #include <drm/drm_managed.h>
34 #include <drm/drm_modeset_helper_vtables.h>
35 #include <drm/drm_print.h>
36 #include <drm/drm_vblank.h>
38 #include "drm_internal.h"
39 #include "drm_trace.h"
42 * DOC: vblank handling
44 * From the computer's perspective, every time the monitor displays
45 * a new frame the scanout engine has "scanned out" the display image
46 * from top to bottom, one row of pixels at a time. The current row
47 * of pixels is referred to as the current scanline.
49 * In addition to the display's visible area, there's usually a couple of
50 * extra scanlines which aren't actually displayed on the screen.
51 * These extra scanlines don't contain image data and are occasionally used
52 * for features like audio and infoframes. The region made up of these
53 * scanlines is referred to as the vertical blanking region, or vblank for
56 * For historical reference, the vertical blanking period was designed to
57 * give the electron gun (on CRTs) enough time to move back to the top of
58 * the screen to start scanning out the next frame. Similar for horizontal
59 * blanking periods. They were designed to give the electron gun enough
60 * time to move back to the other side of the screen to start scanning the
66 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
71 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
72 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
73 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
83 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
84 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
85 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
90 * "Physical top of display" is the reference point for the high-precision/
91 * corrected timestamp.
93 * On a lot of display hardware, programming needs to take effect during the
94 * vertical blanking period so that settings like gamma, the image buffer
95 * buffer to be scanned out, etc. can safely be changed without showing
96 * any visual artifacts on the screen. In some unforgiving hardware, some of
97 * this programming has to both start and end in the same vblank. To help
98 * with the timing of the hardware programming, an interrupt is usually
99 * available to notify the driver when it can start the updating of registers.
100 * The interrupt is in this context named the vblank interrupt.
102 * The vblank interrupt may be fired at different points depending on the
103 * hardware. Some hardware implementations will fire the interrupt when the
104 * new frame start, other implementations will fire the interrupt at different
107 * Vertical blanking plays a major role in graphics rendering. To achieve
108 * tear-free display, users must synchronize page flips and/or rendering to
109 * vertical blanking. The DRM API offers ioctls to perform page flips
110 * synchronized to vertical blanking and wait for vertical blanking.
112 * The DRM core handles most of the vertical blanking management logic, which
113 * involves filtering out spurious interrupts, keeping race-free blanking
114 * counters, coping with counter wrap-around and resets and keeping use counts.
115 * It relies on the driver to generate vertical blanking interrupts and
116 * optionally provide a hardware vertical blanking counter.
118 * Drivers must initialize the vertical blanking handling core with a call to
119 * drm_vblank_init(). Minimally, a driver needs to implement
120 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
121 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
124 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
125 * themselves (for instance to handle page flipping operations). The DRM core
126 * maintains a vertical blanking use count to ensure that the interrupts are not
127 * disabled while a user still needs them. To increment the use count, drivers
128 * call drm_crtc_vblank_get() and release the vblank reference again with
129 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
130 * guaranteed to be enabled.
132 * On many hardware disabling the vblank interrupt cannot be done in a race-free
133 * manner, see &drm_driver.vblank_disable_immediate and
134 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
135 * vblanks after a timer has expired, which can be configured through the
136 * ``vblankoffdelay`` module parameter.
138 * Drivers for hardware without support for vertical-blanking interrupts
139 * must not call drm_vblank_init(). For such drivers, atomic helpers will
140 * automatically generate fake vblank events as part of the display update.
141 * This functionality also can be controlled by the driver by enabling and
142 * disabling struct drm_crtc_state.no_vblank.
145 /* Retry timestamp calculation up to 3 times to satisfy
146 * drm_timestamp_precision before giving up.
148 #define DRM_TIMESTAMP_MAXRETRIES 3
150 /* Threshold in nanoseconds for detection of redundant
151 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
156 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
157 ktime_t *tvblank, bool in_vblank_irq);
159 static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
161 static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
163 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
164 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
165 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
166 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168 static void store_vblank(struct drm_device *dev, unsigned int pipe,
169 u32 vblank_count_inc,
170 ktime_t t_vblank, u32 last)
172 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174 assert_spin_locked(&dev->vblank_time_lock);
178 write_seqlock(&vblank->seqlock);
179 vblank->time = t_vblank;
180 atomic64_add(vblank_count_inc, &vblank->count);
181 write_sequnlock(&vblank->seqlock);
184 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188 return vblank->max_vblank_count ?: dev->max_vblank_count;
192 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
193 * if there is no useable hardware frame counter available.
195 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
201 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
204 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
206 if (drm_WARN_ON(dev, !crtc))
209 if (crtc->funcs->get_vblank_counter)
210 return crtc->funcs->get_vblank_counter(crtc);
211 } else if (dev->driver->get_vblank_counter) {
212 return dev->driver->get_vblank_counter(dev, pipe);
215 return drm_vblank_no_hw_counter(dev, pipe);
219 * Reset the stored timestamp for the current vblank count to correspond
220 * to the last vblank occurred.
222 * Only to be called from drm_crtc_vblank_on().
224 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
225 * device vblank fields.
227 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
232 int count = DRM_TIMESTAMP_MAXRETRIES;
234 spin_lock(&dev->vblank_time_lock);
237 * sample the current counter to avoid random jumps
238 * when drm_vblank_enable() applies the diff
241 cur_vblank = __get_vblank_counter(dev, pipe);
242 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
243 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
246 * Only reinitialize corresponding vblank timestamp if high-precision query
247 * available and didn't fail. Otherwise reinitialize delayed at next vblank
248 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
254 * +1 to make sure user will never see the same
255 * vblank counter value before and after a modeset
257 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
259 spin_unlock(&dev->vblank_time_lock);
263 * Call back into the driver to update the appropriate vblank counter
264 * (specified by @pipe). Deal with wraparound, if it occurred, and
265 * update the last read value so we can deal with wraparound on the next
268 * Only necessary when going from off->on, to account for frames we
269 * didn't get an interrupt for.
271 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
272 * device vblank fields.
274 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
277 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
278 u32 cur_vblank, diff;
281 int count = DRM_TIMESTAMP_MAXRETRIES;
282 int framedur_ns = vblank->framedur_ns;
283 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
286 * Interrupts were disabled prior to this call, so deal with counter
288 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
289 * here if the register is small or we had vblank interrupts off for
292 * We repeat the hardware vblank counter & timestamp query until
293 * we get consistent results. This to prevent races between gpu
294 * updating its hardware counter while we are retrieving the
295 * corresponding vblank timestamp.
298 cur_vblank = __get_vblank_counter(dev, pipe);
299 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
300 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
302 if (max_vblank_count) {
303 /* trust the hw counter when it's around */
304 diff = (cur_vblank - vblank->last) & max_vblank_count;
305 } else if (rc && framedur_ns) {
306 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
309 * Figure out how many vblanks we've missed based
310 * on the difference in the timestamps and the
311 * frame/field duration.
314 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
315 " diff_ns = %lld, framedur_ns = %d)\n",
316 pipe, (long long)diff_ns, framedur_ns);
318 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
320 if (diff == 0 && in_vblank_irq)
321 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
324 /* some kind of default for drivers w/o accurate vbl timestamping */
325 diff = in_vblank_irq ? 1 : 0;
329 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
330 * interval? If so then vblank irqs keep running and it will likely
331 * happen that the hardware vblank counter is not trustworthy as it
332 * might reset at some point in that interval and vblank timestamps
333 * are not trustworthy either in that interval. Iow. this can result
334 * in a bogus diff >> 1 which must be avoided as it would cause
335 * random large forward jumps of the software vblank counter.
337 if (diff > 1 && (vblank->inmodeset & 0x2)) {
339 "clamping vblank bump to 1 on crtc %u: diffr=%u"
340 " due to pre-modeset.\n", pipe, diff);
344 drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
345 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
346 pipe, (unsigned long long)atomic64_read(&vblank->count),
347 diff, cur_vblank, vblank->last);
350 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
355 * Only reinitialize corresponding vblank timestamp if high-precision query
356 * available and didn't fail, or we were called from the vblank interrupt.
357 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
358 * for now, to mark the vblanktimestamp as invalid.
360 if (!rc && !in_vblank_irq)
363 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
366 static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
368 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
371 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
374 count = atomic64_read(&vblank->count);
377 * This read barrier corresponds to the implicit write barrier of the
378 * write seqlock in store_vblank(). Note that this is the only place
379 * where we need an explicit barrier, since all other access goes
380 * through drm_vblank_count_and_time(), which already has the required
381 * read barrier curtesy of the read seqlock.
389 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
390 * @crtc: which counter to retrieve
392 * This function is similar to drm_crtc_vblank_count() but this function
393 * interpolates to handle a race with vblank interrupts using the high precision
394 * timestamping support.
396 * This is mostly useful for hardware that can obtain the scanout position, but
397 * doesn't have a hardware frame counter.
399 u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
401 struct drm_device *dev = crtc->dev;
402 unsigned int pipe = drm_crtc_index(crtc);
406 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
407 !crtc->funcs->get_vblank_timestamp,
408 "This function requires support for accurate vblank timestamps.");
410 spin_lock_irqsave(&dev->vblank_time_lock, flags);
412 drm_update_vblank_count(dev, pipe, false);
413 vblank = drm_vblank_count(dev, pipe);
415 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
419 EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
421 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
423 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
424 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
426 if (drm_WARN_ON(dev, !crtc))
429 if (crtc->funcs->disable_vblank)
430 crtc->funcs->disable_vblank(crtc);
432 dev->driver->disable_vblank(dev, pipe);
437 * Disable vblank irq's on crtc, make sure that last vblank count
438 * of hardware and corresponding consistent software vblank counter
439 * are preserved, even if there are any spurious vblank irq's after
442 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
444 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
445 unsigned long irqflags;
447 assert_spin_locked(&dev->vbl_lock);
449 /* Prevent vblank irq processing while disabling vblank irqs,
450 * so no updates of timestamps or count can happen after we've
451 * disabled. Needed to prevent races in case of delayed irq's.
453 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
456 * Update vblank count and disable vblank interrupts only if the
457 * interrupts were enabled. This avoids calling the ->disable_vblank()
458 * operation in atomic context with the hardware potentially runtime
461 if (!vblank->enabled)
465 * Update the count and timestamp to maintain the
466 * appearance that the counter has been ticking all along until
467 * this time. This makes the count account for the entire time
468 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
470 drm_update_vblank_count(dev, pipe, false);
471 __disable_vblank(dev, pipe);
472 vblank->enabled = false;
475 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
478 static void vblank_disable_fn(struct timer_list *t)
480 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
481 struct drm_device *dev = vblank->dev;
482 unsigned int pipe = vblank->pipe;
483 unsigned long irqflags;
485 spin_lock_irqsave(&dev->vbl_lock, irqflags);
486 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
487 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
488 drm_vblank_disable_and_save(dev, pipe);
490 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
493 static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
497 for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
498 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
500 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
501 drm_core_check_feature(dev, DRIVER_MODESET));
503 del_timer_sync(&vblank->disable_timer);
508 * drm_vblank_init - initialize vblank support
510 * @num_crtcs: number of CRTCs supported by @dev
512 * This function initializes vblank support for @num_crtcs display pipelines.
513 * Cleanup is handled automatically through a cleanup function added with
517 * Zero on success or a negative error code on failure.
519 int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
524 spin_lock_init(&dev->vbl_lock);
525 spin_lock_init(&dev->vblank_time_lock);
527 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
531 dev->num_crtcs = num_crtcs;
533 ret = drmm_add_action(dev, drm_vblank_init_release, NULL);
537 for (i = 0; i < num_crtcs; i++) {
538 struct drm_vblank_crtc *vblank = &dev->vblank[i];
542 init_waitqueue_head(&vblank->queue);
543 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
544 seqlock_init(&vblank->seqlock);
549 EXPORT_SYMBOL(drm_vblank_init);
552 * drm_dev_has_vblank - test if vblanking has been initialized for
556 * Drivers may call this function to test if vblank support is
557 * initialized for a device. For most hardware this means that vblanking
558 * can also be enabled.
560 * Atomic helpers use this function to initialize
561 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
564 * True if vblanking has been initialized for the given device, false
567 bool drm_dev_has_vblank(const struct drm_device *dev)
569 return dev->num_crtcs != 0;
571 EXPORT_SYMBOL(drm_dev_has_vblank);
574 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
575 * @crtc: which CRTC's vblank waitqueue to retrieve
577 * This function returns a pointer to the vblank waitqueue for the CRTC.
578 * Drivers can use this to implement vblank waits using wait_event() and related
581 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
583 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
585 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
589 * drm_calc_timestamping_constants - calculate vblank timestamp constants
590 * @crtc: drm_crtc whose timestamp constants should be updated.
591 * @mode: display mode containing the scanout timings
593 * Calculate and store various constants which are later needed by vblank and
594 * swap-completion timestamping, e.g, by
595 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
596 * CRTC's true scanout timing, so they take things like panel scaling or
597 * other adjustments into account.
599 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
600 const struct drm_display_mode *mode)
602 struct drm_device *dev = crtc->dev;
603 unsigned int pipe = drm_crtc_index(crtc);
604 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
605 int linedur_ns = 0, framedur_ns = 0;
606 int dotclock = mode->crtc_clock;
608 if (!drm_dev_has_vblank(dev))
611 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
614 /* Valid dotclock? */
616 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
619 * Convert scanline length in pixels and video
620 * dot clock to line duration and frame duration
623 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
624 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
627 * Fields of interlaced scanout modes are only half a frame duration.
629 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
632 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
636 vblank->linedur_ns = linedur_ns;
637 vblank->framedur_ns = framedur_ns;
638 vblank->hwmode = *mode;
641 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
642 crtc->base.id, mode->crtc_htotal,
643 mode->crtc_vtotal, mode->crtc_vdisplay);
644 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
645 crtc->base.id, dotclock, framedur_ns, linedur_ns);
647 EXPORT_SYMBOL(drm_calc_timestamping_constants);
650 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
652 * @crtc: CRTC whose vblank timestamp to retrieve
653 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
654 * On return contains true maximum error of timestamp
655 * @vblank_time: Pointer to time which should receive the timestamp
657 * True when called from drm_crtc_handle_vblank(). Some drivers
658 * need to apply some workarounds for gpu-specific vblank irq quirks
660 * @get_scanout_position:
661 * Callback function to retrieve the scanout position. See
662 * @struct drm_crtc_helper_funcs.get_scanout_position.
664 * Implements calculation of exact vblank timestamps from given drm_display_mode
665 * timings and current video scanout position of a CRTC.
667 * The current implementation only handles standard video modes. For double scan
668 * and interlaced modes the driver is supposed to adjust the hardware mode
669 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
670 * match the scanout position reported.
672 * Note that atomic drivers must call drm_calc_timestamping_constants() before
673 * enabling a CRTC. The atomic helpers already take care of that in
674 * drm_atomic_helper_update_legacy_modeset_state().
678 * Returns true on success, and false on failure, i.e. when no accurate
679 * timestamp could be acquired.
682 drm_crtc_vblank_helper_get_vblank_timestamp_internal(
683 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
685 drm_vblank_get_scanout_position_func get_scanout_position)
687 struct drm_device *dev = crtc->dev;
688 unsigned int pipe = crtc->index;
689 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
690 struct timespec64 ts_etime, ts_vblank_time;
691 ktime_t stime, etime;
693 const struct drm_display_mode *mode;
695 int delta_ns, duration_ns;
697 if (pipe >= dev->num_crtcs) {
698 drm_err(dev, "Invalid crtc %u\n", pipe);
702 /* Scanout position query not supported? Should not happen. */
703 if (!get_scanout_position) {
704 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
708 if (drm_drv_uses_atomic_modeset(dev))
709 mode = &vblank->hwmode;
711 mode = &crtc->hwmode;
713 /* If mode timing undefined, just return as no-op:
714 * Happens during initial modesetting of a crtc.
716 if (mode->crtc_clock == 0) {
717 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
719 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
723 /* Get current scanout position with system timestamp.
724 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
725 * if single query takes longer than max_error nanoseconds.
727 * This guarantees a tight bound on maximum error if
728 * code gets preempted or delayed for some reason.
730 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
732 * Get vertical and horizontal scanout position vpos, hpos,
733 * and bounding timestamps stime, etime, pre/post query.
735 vbl_status = get_scanout_position(crtc, in_vblank_irq,
740 /* Return as no-op if scanout query unsupported or failed. */
743 "crtc %u : scanoutpos query failed.\n",
748 /* Compute uncertainty in timestamp of scanout position query. */
749 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
751 /* Accept result with < max_error nsecs timing uncertainty. */
752 if (duration_ns <= *max_error)
756 /* Noisy system timing? */
757 if (i == DRM_TIMESTAMP_MAXRETRIES) {
759 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
760 pipe, duration_ns / 1000, *max_error / 1000, i);
763 /* Return upper bound of timestamp precision error. */
764 *max_error = duration_ns;
766 /* Convert scanout position into elapsed time at raw_time query
767 * since start of scanout at first display scanline. delta_ns
768 * can be negative if start of scanout hasn't happened yet.
770 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
773 /* Subtract time delta from raw timestamp to get final
774 * vblank_time timestamp for end of vblank.
776 *vblank_time = ktime_sub_ns(etime, delta_ns);
778 if (!drm_debug_enabled(DRM_UT_VBL))
781 ts_etime = ktime_to_timespec64(etime);
782 ts_vblank_time = ktime_to_timespec64(*vblank_time);
785 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
787 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
788 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
789 duration_ns / 1000, i);
793 EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
796 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
798 * @crtc: CRTC whose vblank timestamp to retrieve
799 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
800 * On return contains true maximum error of timestamp
801 * @vblank_time: Pointer to time which should receive the timestamp
803 * True when called from drm_crtc_handle_vblank(). Some drivers
804 * need to apply some workarounds for gpu-specific vblank irq quirks
807 * Implements calculation of exact vblank timestamps from given drm_display_mode
808 * timings and current video scanout position of a CRTC. This can be directly
809 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
810 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
812 * The current implementation only handles standard video modes. For double scan
813 * and interlaced modes the driver is supposed to adjust the hardware mode
814 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
815 * match the scanout position reported.
817 * Note that atomic drivers must call drm_calc_timestamping_constants() before
818 * enabling a CRTC. The atomic helpers already take care of that in
819 * drm_atomic_helper_update_legacy_modeset_state().
823 * Returns true on success, and false on failure, i.e. when no accurate
824 * timestamp could be acquired.
826 bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
828 ktime_t *vblank_time,
831 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
832 crtc, max_error, vblank_time, in_vblank_irq,
833 crtc->helper_private->get_scanout_position);
835 EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
838 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
841 * @pipe: index of CRTC whose vblank timestamp to retrieve
842 * @tvblank: Pointer to target time which should receive the timestamp
844 * True when called from drm_crtc_handle_vblank(). Some drivers
845 * need to apply some workarounds for gpu-specific vblank irq quirks
848 * Fetches the system timestamp corresponding to the time of the most recent
849 * vblank interval on specified CRTC. May call into kms-driver to
850 * compute the timestamp with a high-precision GPU specific method.
852 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
853 * call, i.e., it isn't very precisely locked to the true vblank.
856 * True if timestamp is considered to be very precise, false otherwise.
859 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
860 ktime_t *tvblank, bool in_vblank_irq)
862 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
865 /* Define requested maximum error on timestamps (nanoseconds). */
866 int max_error = (int) drm_timestamp_precision * 1000;
868 /* Query driver if possible and precision timestamping enabled. */
869 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
870 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
872 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
873 tvblank, in_vblank_irq);
876 /* GPU high precision timestamp query unsupported or failed.
877 * Return current monotonic/gettimeofday timestamp as best estimate.
880 *tvblank = ktime_get();
886 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
887 * @crtc: which counter to retrieve
889 * Fetches the "cooked" vblank count value that represents the number of
890 * vblank events since the system was booted, including lost events due to
891 * modesetting activity. Note that this timer isn't correct against a racing
892 * vblank interrupt (since it only reports the software vblank counter), see
893 * drm_crtc_accurate_vblank_count() for such use-cases.
895 * Note that for a given vblank counter value drm_crtc_handle_vblank()
896 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
897 * provide a barrier: Any writes done before calling
898 * drm_crtc_handle_vblank() will be visible to callers of the later
899 * functions, iff the vblank count is the same or a later one.
901 * See also &drm_vblank_crtc.count.
904 * The software vblank counter.
906 u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
908 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
910 EXPORT_SYMBOL(drm_crtc_vblank_count);
913 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
914 * system timestamp corresponding to that vblank counter value.
916 * @pipe: index of CRTC whose counter to retrieve
917 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
919 * Fetches the "cooked" vblank count value that represents the number of
920 * vblank events since the system was booted, including lost events due to
921 * modesetting activity. Returns corresponding system timestamp of the time
922 * of the vblank interval that corresponds to the current vblank counter value.
924 * This is the legacy version of drm_crtc_vblank_count_and_time().
926 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
929 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
933 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
939 seq = read_seqbegin(&vblank->seqlock);
940 vblank_count = atomic64_read(&vblank->count);
941 *vblanktime = vblank->time;
942 } while (read_seqretry(&vblank->seqlock, seq));
948 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
949 * and the system timestamp corresponding to that vblank counter value
950 * @crtc: which counter to retrieve
951 * @vblanktime: Pointer to time to receive the vblank timestamp.
953 * Fetches the "cooked" vblank count value that represents the number of
954 * vblank events since the system was booted, including lost events due to
955 * modesetting activity. Returns corresponding system timestamp of the time
956 * of the vblank interval that corresponds to the current vblank counter value.
958 * Note that for a given vblank counter value drm_crtc_handle_vblank()
959 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
960 * provide a barrier: Any writes done before calling
961 * drm_crtc_handle_vblank() will be visible to callers of the later
962 * functions, iff the vblank count is the same or a later one.
964 * See also &drm_vblank_crtc.count.
966 u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
969 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
972 EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
974 static void send_vblank_event(struct drm_device *dev,
975 struct drm_pending_vblank_event *e,
976 u64 seq, ktime_t now)
978 struct timespec64 tv;
980 switch (e->event.base.type) {
981 case DRM_EVENT_VBLANK:
982 case DRM_EVENT_FLIP_COMPLETE:
983 tv = ktime_to_timespec64(now);
984 e->event.vbl.sequence = seq;
986 * e->event is a user space structure, with hardcoded unsigned
987 * 32-bit seconds/microseconds. This is safe as we always use
988 * monotonic timestamps since linux-4.15
990 e->event.vbl.tv_sec = tv.tv_sec;
991 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
993 case DRM_EVENT_CRTC_SEQUENCE:
995 e->event.seq.sequence = seq;
996 e->event.seq.time_ns = ktime_to_ns(now);
999 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1000 drm_send_event_locked(dev, &e->base);
1004 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1005 * @crtc: the source CRTC of the vblank event
1006 * @e: the event to send
1008 * A lot of drivers need to generate vblank events for the very next vblank
1009 * interrupt. For example when the page flip interrupt happens when the page
1010 * flip gets armed, but not when it actually executes within the next vblank
1011 * period. This helper function implements exactly the required vblank arming
1014 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1015 * atomic commit must ensure that the next vblank happens at exactly the same
1016 * time as the atomic commit is committed to the hardware. This function itself
1017 * does **not** protect against the next vblank interrupt racing with either this
1018 * function call or the atomic commit operation. A possible sequence could be:
1020 * 1. Driver commits new hardware state into vblank-synchronized registers.
1021 * 2. A vblank happens, committing the hardware state. Also the corresponding
1022 * vblank interrupt is fired off and fully processed by the interrupt
1024 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1025 * 4. The event is only send out for the next vblank, which is wrong.
1027 * An equivalent race can happen when the driver calls
1028 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1030 * The only way to make this work safely is to prevent the vblank from firing
1031 * (and the hardware from committing anything else) until the entire atomic
1032 * commit sequence has run to completion. If the hardware does not have such a
1033 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1034 * Instead drivers need to manually send out the event from their interrupt
1035 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1036 * possible race with the hardware committing the atomic update.
1038 * Caller must hold a vblank reference for the event @e acquired by a
1039 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1041 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1042 struct drm_pending_vblank_event *e)
1044 struct drm_device *dev = crtc->dev;
1045 unsigned int pipe = drm_crtc_index(crtc);
1047 assert_spin_locked(&dev->event_lock);
1050 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1051 list_add_tail(&e->base.link, &dev->vblank_event_list);
1053 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1056 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1057 * @crtc: the source CRTC of the vblank event
1058 * @e: the event to send
1060 * Updates sequence # and timestamp on event for the most recently processed
1061 * vblank, and sends it to userspace. Caller must hold event lock.
1063 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1064 * situation, especially to send out events for atomic commit operations.
1066 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1067 struct drm_pending_vblank_event *e)
1069 struct drm_device *dev = crtc->dev;
1071 unsigned int pipe = drm_crtc_index(crtc);
1074 if (drm_dev_has_vblank(dev)) {
1075 seq = drm_vblank_count_and_time(dev, pipe, &now);
1082 send_vblank_event(dev, e, seq, now);
1084 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1086 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1088 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1089 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1091 if (drm_WARN_ON(dev, !crtc))
1094 if (crtc->funcs->enable_vblank)
1095 return crtc->funcs->enable_vblank(crtc);
1096 } else if (dev->driver->enable_vblank) {
1097 return dev->driver->enable_vblank(dev, pipe);
1103 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1105 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1108 assert_spin_locked(&dev->vbl_lock);
1110 spin_lock(&dev->vblank_time_lock);
1112 if (!vblank->enabled) {
1114 * Enable vblank irqs under vblank_time_lock protection.
1115 * All vblank count & timestamp updates are held off
1116 * until we are done reinitializing master counter and
1117 * timestamps. Filtercode in drm_handle_vblank() will
1118 * prevent double-accounting of same vblank interval.
1120 ret = __enable_vblank(dev, pipe);
1121 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1124 atomic_dec(&vblank->refcount);
1126 drm_update_vblank_count(dev, pipe, 0);
1127 /* drm_update_vblank_count() includes a wmb so we just
1128 * need to ensure that the compiler emits the write
1129 * to mark the vblank as enabled after the call
1130 * to drm_update_vblank_count().
1132 WRITE_ONCE(vblank->enabled, true);
1136 spin_unlock(&dev->vblank_time_lock);
1141 static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1143 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1144 unsigned long irqflags;
1147 if (!drm_dev_has_vblank(dev))
1150 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1153 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1154 /* Going from 0->1 means we have to enable interrupts again */
1155 if (atomic_add_return(1, &vblank->refcount) == 1) {
1156 ret = drm_vblank_enable(dev, pipe);
1158 if (!vblank->enabled) {
1159 atomic_dec(&vblank->refcount);
1163 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1169 * drm_crtc_vblank_get - get a reference count on vblank events
1170 * @crtc: which CRTC to own
1172 * Acquire a reference count on vblank events to avoid having them disabled
1176 * Zero on success or a negative error code on failure.
1178 int drm_crtc_vblank_get(struct drm_crtc *crtc)
1180 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1182 EXPORT_SYMBOL(drm_crtc_vblank_get);
1184 static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1186 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1188 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1191 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1194 /* Last user schedules interrupt disable */
1195 if (atomic_dec_and_test(&vblank->refcount)) {
1196 if (drm_vblank_offdelay == 0)
1198 else if (drm_vblank_offdelay < 0)
1199 vblank_disable_fn(&vblank->disable_timer);
1200 else if (!dev->vblank_disable_immediate)
1201 mod_timer(&vblank->disable_timer,
1202 jiffies + ((drm_vblank_offdelay * HZ)/1000));
1207 * drm_crtc_vblank_put - give up ownership of vblank events
1208 * @crtc: which counter to give up
1210 * Release ownership of a given vblank counter, turning off interrupts
1211 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1213 void drm_crtc_vblank_put(struct drm_crtc *crtc)
1215 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1217 EXPORT_SYMBOL(drm_crtc_vblank_put);
1220 * drm_wait_one_vblank - wait for one vblank
1224 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1225 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1226 * due to lack of driver support or because the crtc is off.
1228 * This is the legacy version of drm_crtc_wait_one_vblank().
1230 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1232 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1236 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1239 ret = drm_vblank_get(dev, pipe);
1240 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1244 last = drm_vblank_count(dev, pipe);
1246 ret = wait_event_timeout(vblank->queue,
1247 last != drm_vblank_count(dev, pipe),
1248 msecs_to_jiffies(100));
1250 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1252 drm_vblank_put(dev, pipe);
1254 EXPORT_SYMBOL(drm_wait_one_vblank);
1257 * drm_crtc_wait_one_vblank - wait for one vblank
1260 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1261 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1262 * due to lack of driver support or because the crtc is off.
1264 void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1266 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1268 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1271 * drm_crtc_vblank_off - disable vblank events on a CRTC
1272 * @crtc: CRTC in question
1274 * Drivers can use this function to shut down the vblank interrupt handling when
1275 * disabling a crtc. This function ensures that the latest vblank frame count is
1276 * stored so that drm_vblank_on can restore it again.
1278 * Drivers must use this function when the hardware vblank counter can get
1279 * reset, e.g. when suspending or disabling the @crtc in general.
1281 void drm_crtc_vblank_off(struct drm_crtc *crtc)
1283 struct drm_device *dev = crtc->dev;
1284 unsigned int pipe = drm_crtc_index(crtc);
1285 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1286 struct drm_pending_vblank_event *e, *t;
1289 unsigned long irqflags;
1292 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1295 spin_lock_irqsave(&dev->event_lock, irqflags);
1297 spin_lock(&dev->vbl_lock);
1298 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1299 pipe, vblank->enabled, vblank->inmodeset);
1301 /* Avoid redundant vblank disables without previous
1302 * drm_crtc_vblank_on(). */
1303 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1304 drm_vblank_disable_and_save(dev, pipe);
1306 wake_up(&vblank->queue);
1309 * Prevent subsequent drm_vblank_get() from re-enabling
1310 * the vblank interrupt by bumping the refcount.
1312 if (!vblank->inmodeset) {
1313 atomic_inc(&vblank->refcount);
1314 vblank->inmodeset = 1;
1316 spin_unlock(&dev->vbl_lock);
1318 /* Send any queued vblank events, lest the natives grow disquiet */
1319 seq = drm_vblank_count_and_time(dev, pipe, &now);
1321 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1322 if (e->pipe != pipe)
1324 drm_dbg_core(dev, "Sending premature vblank event on disable: "
1325 "wanted %llu, current %llu\n",
1327 list_del(&e->base.link);
1328 drm_vblank_put(dev, pipe);
1329 send_vblank_event(dev, e, seq, now);
1331 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1333 /* Will be reset by the modeset helpers when re-enabling the crtc by
1334 * calling drm_calc_timestamping_constants(). */
1335 vblank->hwmode.crtc_clock = 0;
1337 EXPORT_SYMBOL(drm_crtc_vblank_off);
1340 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1341 * @crtc: CRTC in question
1343 * Drivers can use this function to reset the vblank state to off at load time.
1344 * Drivers should use this together with the drm_crtc_vblank_off() and
1345 * drm_crtc_vblank_on() functions. The difference compared to
1346 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1347 * and hence doesn't need to call any driver hooks.
1349 * This is useful for recovering driver state e.g. on driver load, or on resume.
1351 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1353 struct drm_device *dev = crtc->dev;
1354 unsigned long irqflags;
1355 unsigned int pipe = drm_crtc_index(crtc);
1356 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1358 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1360 * Prevent subsequent drm_vblank_get() from enabling the vblank
1361 * interrupt by bumping the refcount.
1363 if (!vblank->inmodeset) {
1364 atomic_inc(&vblank->refcount);
1365 vblank->inmodeset = 1;
1367 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1369 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1371 EXPORT_SYMBOL(drm_crtc_vblank_reset);
1374 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1375 * @crtc: CRTC in question
1376 * @max_vblank_count: max hardware vblank counter value
1378 * Update the maximum hardware vblank counter value for @crtc
1379 * at runtime. Useful for hardware where the operation of the
1380 * hardware vblank counter depends on the currently active
1381 * display configuration.
1383 * For example, if the hardware vblank counter does not work
1384 * when a specific connector is active the maximum can be set
1385 * to zero. And when that specific connector isn't active the
1386 * maximum can again be set to the appropriate non-zero value.
1388 * If used, must be called before drm_vblank_on().
1390 void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1391 u32 max_vblank_count)
1393 struct drm_device *dev = crtc->dev;
1394 unsigned int pipe = drm_crtc_index(crtc);
1395 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1397 drm_WARN_ON(dev, dev->max_vblank_count);
1398 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1400 vblank->max_vblank_count = max_vblank_count;
1402 EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1405 * drm_crtc_vblank_on - enable vblank events on a CRTC
1406 * @crtc: CRTC in question
1408 * This functions restores the vblank interrupt state captured with
1409 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1410 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1411 * unbalanced and so can also be unconditionally called in driver load code to
1412 * reflect the current hardware state of the crtc.
1414 void drm_crtc_vblank_on(struct drm_crtc *crtc)
1416 struct drm_device *dev = crtc->dev;
1417 unsigned int pipe = drm_crtc_index(crtc);
1418 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1419 unsigned long irqflags;
1421 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1424 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1425 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1426 pipe, vblank->enabled, vblank->inmodeset);
1428 /* Drop our private "prevent drm_vblank_get" refcount */
1429 if (vblank->inmodeset) {
1430 atomic_dec(&vblank->refcount);
1431 vblank->inmodeset = 0;
1434 drm_reset_vblank_timestamp(dev, pipe);
1437 * re-enable interrupts if there are users left, or the
1438 * user wishes vblank interrupts to be enabled all the time.
1440 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1441 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1442 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1444 EXPORT_SYMBOL(drm_crtc_vblank_on);
1447 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1451 * Power manamement features can cause frame counter resets between vblank
1452 * disable and enable. Drivers can use this function in their
1453 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1454 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1457 * This function is the legacy version of drm_crtc_vblank_restore().
1459 void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1462 struct drm_vblank_crtc *vblank;
1465 u32 cur_vblank, diff = 1;
1466 int count = DRM_TIMESTAMP_MAXRETRIES;
1468 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1471 assert_spin_locked(&dev->vbl_lock);
1472 assert_spin_locked(&dev->vblank_time_lock);
1474 vblank = &dev->vblank[pipe];
1476 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1477 "Cannot compute missed vblanks without frame duration\n");
1478 framedur_ns = vblank->framedur_ns;
1481 cur_vblank = __get_vblank_counter(dev, pipe);
1482 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1483 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1485 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1487 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1491 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1492 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1493 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1495 EXPORT_SYMBOL(drm_vblank_restore);
1498 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1499 * @crtc: CRTC in question
1501 * Power manamement features can cause frame counter resets between vblank
1502 * disable and enable. Drivers can use this function in their
1503 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1504 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1507 void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1509 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1511 EXPORT_SYMBOL(drm_crtc_vblank_restore);
1513 static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1516 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1518 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1519 if (!drm_dev_has_vblank(dev))
1522 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1526 * To avoid all the problems that might happen if interrupts
1527 * were enabled/disabled around or between these calls, we just
1528 * have the kernel take a reference on the CRTC (just once though
1529 * to avoid corrupting the count if multiple, mismatch calls occur),
1530 * so that interrupts remain enabled in the interim.
1532 if (!vblank->inmodeset) {
1533 vblank->inmodeset = 0x1;
1534 if (drm_vblank_get(dev, pipe) == 0)
1535 vblank->inmodeset |= 0x2;
1539 static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1542 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1543 unsigned long irqflags;
1545 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1546 if (!drm_dev_has_vblank(dev))
1549 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1552 if (vblank->inmodeset) {
1553 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1554 drm_reset_vblank_timestamp(dev, pipe);
1555 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1557 if (vblank->inmodeset & 0x2)
1558 drm_vblank_put(dev, pipe);
1560 vblank->inmodeset = 0;
1564 int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1565 struct drm_file *file_priv)
1567 struct drm_modeset_ctl *modeset = data;
1570 /* If drm_vblank_init() hasn't been called yet, just no-op */
1571 if (!drm_dev_has_vblank(dev))
1574 /* KMS drivers handle this internally */
1575 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1578 pipe = modeset->crtc;
1579 if (pipe >= dev->num_crtcs)
1582 switch (modeset->cmd) {
1583 case _DRM_PRE_MODESET:
1584 drm_legacy_vblank_pre_modeset(dev, pipe);
1586 case _DRM_POST_MODESET:
1587 drm_legacy_vblank_post_modeset(dev, pipe);
1596 static inline bool vblank_passed(u64 seq, u64 ref)
1598 return (seq - ref) <= (1 << 23);
1601 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1603 union drm_wait_vblank *vblwait,
1604 struct drm_file *file_priv)
1606 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1607 struct drm_pending_vblank_event *e;
1609 unsigned long flags;
1613 e = kzalloc(sizeof(*e), GFP_KERNEL);
1620 e->event.base.type = DRM_EVENT_VBLANK;
1621 e->event.base.length = sizeof(e->event.vbl);
1622 e->event.vbl.user_data = vblwait->request.signal;
1623 e->event.vbl.crtc_id = 0;
1624 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1625 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1627 e->event.vbl.crtc_id = crtc->base.id;
1630 spin_lock_irqsave(&dev->event_lock, flags);
1633 * drm_crtc_vblank_off() might have been called after we called
1634 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1635 * vblank disable, so no need for further locking. The reference from
1636 * drm_vblank_get() protects against vblank disable from another source.
1638 if (!READ_ONCE(vblank->enabled)) {
1643 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1649 seq = drm_vblank_count_and_time(dev, pipe, &now);
1651 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1652 req_seq, seq, pipe);
1654 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1656 e->sequence = req_seq;
1657 if (vblank_passed(seq, req_seq)) {
1658 drm_vblank_put(dev, pipe);
1659 send_vblank_event(dev, e, seq, now);
1660 vblwait->reply.sequence = seq;
1662 /* drm_handle_vblank_events will call drm_vblank_put */
1663 list_add_tail(&e->base.link, &dev->vblank_event_list);
1664 vblwait->reply.sequence = req_seq;
1667 spin_unlock_irqrestore(&dev->event_lock, flags);
1672 spin_unlock_irqrestore(&dev->event_lock, flags);
1675 drm_vblank_put(dev, pipe);
1679 static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1681 if (vblwait->request.sequence)
1684 return _DRM_VBLANK_RELATIVE ==
1685 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1687 _DRM_VBLANK_NEXTONMISS));
1691 * Widen a 32-bit param to 64-bits.
1693 * \param narrow 32-bit value (missing upper 32 bits)
1694 * \param near 64-bit value that should be 'close' to near
1696 * This function returns a 64-bit value using the lower 32-bits from
1697 * 'narrow' and constructing the upper 32-bits so that the result is
1698 * as close as possible to 'near'.
1701 static u64 widen_32_to_64(u32 narrow, u64 near)
1703 return near + (s32) (narrow - near);
1706 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1707 struct drm_wait_vblank_reply *reply)
1710 struct timespec64 ts;
1713 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1714 * to store the seconds. This is safe as we always use monotonic
1715 * timestamps since linux-4.15.
1717 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1718 ts = ktime_to_timespec64(now);
1719 reply->tval_sec = (u32)ts.tv_sec;
1720 reply->tval_usec = ts.tv_nsec / 1000;
1723 int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1724 struct drm_file *file_priv)
1726 struct drm_crtc *crtc;
1727 struct drm_vblank_crtc *vblank;
1728 union drm_wait_vblank *vblwait = data;
1731 unsigned int pipe_index;
1732 unsigned int flags, pipe, high_pipe;
1734 if (!dev->irq_enabled)
1737 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1740 if (vblwait->request.type &
1741 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1742 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1744 "Unsupported type value 0x%x, supported mask 0x%x\n",
1745 vblwait->request.type,
1746 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1747 _DRM_VBLANK_HIGH_CRTC_MASK));
1751 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1752 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1754 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1756 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1758 /* Convert lease-relative crtc index into global crtc index */
1759 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1761 drm_for_each_crtc(crtc, dev) {
1762 if (drm_lease_held(file_priv, crtc->base.id)) {
1763 if (pipe_index == 0)
1773 if (pipe >= dev->num_crtcs)
1776 vblank = &dev->vblank[pipe];
1778 /* If the counter is currently enabled and accurate, short-circuit
1779 * queries to return the cached timestamp of the last vblank.
1781 if (dev->vblank_disable_immediate &&
1782 drm_wait_vblank_is_query(vblwait) &&
1783 READ_ONCE(vblank->enabled)) {
1784 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1788 ret = drm_vblank_get(dev, pipe);
1791 "crtc %d failed to acquire vblank counter, %d\n",
1795 seq = drm_vblank_count(dev, pipe);
1797 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1798 case _DRM_VBLANK_RELATIVE:
1799 req_seq = seq + vblwait->request.sequence;
1800 vblwait->request.sequence = req_seq;
1801 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1803 case _DRM_VBLANK_ABSOLUTE:
1804 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1811 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1812 vblank_passed(seq, req_seq)) {
1814 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1815 vblwait->request.sequence = req_seq;
1818 if (flags & _DRM_VBLANK_EVENT) {
1819 /* must hold on to the vblank ref until the event fires
1820 * drm_vblank_put will be called asynchronously
1822 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1825 if (req_seq != seq) {
1828 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1830 wait = wait_event_interruptible_timeout(vblank->queue,
1831 vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1832 !READ_ONCE(vblank->enabled),
1833 msecs_to_jiffies(3000));
1841 /* interrupted by signal */
1850 if (ret != -EINTR) {
1851 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1853 drm_dbg_core(dev, "crtc %d returning %u to client\n",
1854 pipe, vblwait->reply.sequence);
1856 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1861 drm_vblank_put(dev, pipe);
1865 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1867 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1868 bool high_prec = false;
1869 struct drm_pending_vblank_event *e, *t;
1873 assert_spin_locked(&dev->event_lock);
1875 seq = drm_vblank_count_and_time(dev, pipe, &now);
1877 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1878 if (e->pipe != pipe)
1880 if (!vblank_passed(seq, e->sequence))
1883 drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1886 list_del(&e->base.link);
1887 drm_vblank_put(dev, pipe);
1888 send_vblank_event(dev, e, seq, now);
1891 if (crtc && crtc->funcs->get_vblank_timestamp)
1894 trace_drm_vblank_event(pipe, seq, now, high_prec);
1898 * drm_handle_vblank - handle a vblank event
1900 * @pipe: index of CRTC where this event occurred
1902 * Drivers should call this routine in their vblank interrupt handlers to
1903 * update the vblank counter and send any signals that may be pending.
1905 * This is the legacy version of drm_crtc_handle_vblank().
1907 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1909 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1910 unsigned long irqflags;
1913 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1916 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1919 spin_lock_irqsave(&dev->event_lock, irqflags);
1921 /* Need timestamp lock to prevent concurrent execution with
1922 * vblank enable/disable, as this would cause inconsistent
1923 * or corrupted timestamps and vblank counts.
1925 spin_lock(&dev->vblank_time_lock);
1927 /* Vblank irq handling disabled. Nothing to do. */
1928 if (!vblank->enabled) {
1929 spin_unlock(&dev->vblank_time_lock);
1930 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1934 drm_update_vblank_count(dev, pipe, true);
1936 spin_unlock(&dev->vblank_time_lock);
1938 wake_up(&vblank->queue);
1940 /* With instant-off, we defer disabling the interrupt until after
1941 * we finish processing the following vblank after all events have
1942 * been signaled. The disable has to be last (after
1943 * drm_handle_vblank_events) so that the timestamp is always accurate.
1945 disable_irq = (dev->vblank_disable_immediate &&
1946 drm_vblank_offdelay > 0 &&
1947 !atomic_read(&vblank->refcount));
1949 drm_handle_vblank_events(dev, pipe);
1951 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1954 vblank_disable_fn(&vblank->disable_timer);
1958 EXPORT_SYMBOL(drm_handle_vblank);
1961 * drm_crtc_handle_vblank - handle a vblank event
1962 * @crtc: where this event occurred
1964 * Drivers should call this routine in their vblank interrupt handlers to
1965 * update the vblank counter and send any signals that may be pending.
1967 * This is the native KMS version of drm_handle_vblank().
1969 * Note that for a given vblank counter value drm_crtc_handle_vblank()
1970 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1971 * provide a barrier: Any writes done before calling
1972 * drm_crtc_handle_vblank() will be visible to callers of the later
1973 * functions, iff the vblank count is the same or a later one.
1975 * See also &drm_vblank_crtc.count.
1978 * True if the event was successfully handled, false on failure.
1980 bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1982 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1984 EXPORT_SYMBOL(drm_crtc_handle_vblank);
1987 * Get crtc VBLANK count.
1989 * \param dev DRM device
1990 * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1991 * \param file_priv drm file private for the user's open file descriptor
1994 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1995 struct drm_file *file_priv)
1997 struct drm_crtc *crtc;
1998 struct drm_vblank_crtc *vblank;
2000 struct drm_crtc_get_sequence *get_seq = data;
2002 bool vblank_enabled;
2005 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2008 if (!dev->irq_enabled)
2011 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2015 pipe = drm_crtc_index(crtc);
2017 vblank = &dev->vblank[pipe];
2018 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
2020 if (!vblank_enabled) {
2021 ret = drm_crtc_vblank_get(crtc);
2024 "crtc %d failed to acquire vblank counter, %d\n",
2029 drm_modeset_lock(&crtc->mutex, NULL);
2031 get_seq->active = crtc->state->enable;
2033 get_seq->active = crtc->enabled;
2034 drm_modeset_unlock(&crtc->mutex);
2035 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2036 get_seq->sequence_ns = ktime_to_ns(now);
2037 if (!vblank_enabled)
2038 drm_crtc_vblank_put(crtc);
2043 * Queue a event for VBLANK sequence
2045 * \param dev DRM device
2046 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
2047 * \param file_priv drm file private for the user's open file descriptor
2050 int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2051 struct drm_file *file_priv)
2053 struct drm_crtc *crtc;
2054 struct drm_vblank_crtc *vblank;
2056 struct drm_crtc_queue_sequence *queue_seq = data;
2058 struct drm_pending_vblank_event *e;
2063 unsigned long spin_flags;
2065 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2068 if (!dev->irq_enabled)
2071 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2075 flags = queue_seq->flags;
2076 /* Check valid flag bits */
2077 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2078 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2081 pipe = drm_crtc_index(crtc);
2083 vblank = &dev->vblank[pipe];
2085 e = kzalloc(sizeof(*e), GFP_KERNEL);
2089 ret = drm_crtc_vblank_get(crtc);
2092 "crtc %d failed to acquire vblank counter, %d\n",
2097 seq = drm_vblank_count_and_time(dev, pipe, &now);
2098 req_seq = queue_seq->sequence;
2100 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2103 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
2107 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2108 e->event.base.length = sizeof(e->event.seq);
2109 e->event.seq.user_data = queue_seq->user_data;
2111 spin_lock_irqsave(&dev->event_lock, spin_flags);
2114 * drm_crtc_vblank_off() might have been called after we called
2115 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2116 * vblank disable, so no need for further locking. The reference from
2117 * drm_crtc_vblank_get() protects against vblank disable from another source.
2119 if (!READ_ONCE(vblank->enabled)) {
2124 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2130 e->sequence = req_seq;
2132 if (vblank_passed(seq, req_seq)) {
2133 drm_crtc_vblank_put(crtc);
2134 send_vblank_event(dev, e, seq, now);
2135 queue_seq->sequence = seq;
2137 /* drm_handle_vblank_events will call drm_vblank_put */
2138 list_add_tail(&e->base.link, &dev->vblank_event_list);
2139 queue_seq->sequence = req_seq;
2142 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
2146 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
2147 drm_crtc_vblank_put(crtc);