1/*
2 * drm_irq.c IRQ and vblank support
3 *
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include <linux/export.h>
28#include <linux/kthread.h>
29#include <linux/moduleparam.h>
30
31#include <drm/drm_crtc.h>
32#include <drm/drm_drv.h>
33#include <drm/drm_framebuffer.h>
34#include <drm/drm_managed.h>
35#include <drm/drm_modeset_helper_vtables.h>
36#include <drm/drm_print.h>
37#include <drm/drm_vblank.h>
38
39#include "drm_internal.h"
40#include "drm_trace.h"
41
42/**
43 * DOC: vblank handling
44 *
45 * From the computer's perspective, every time the monitor displays
46 * a new frame the scanout engine has "scanned out" the display image
47 * from top to bottom, one row of pixels at a time. The current row
48 * of pixels is referred to as the current scanline.
49 *
50 * In addition to the display's visible area, there's usually a couple of
51 * extra scanlines which aren't actually displayed on the screen.
52 * These extra scanlines don't contain image data and are occasionally used
53 * for features like audio and infoframes. The region made up of these
54 * scanlines is referred to as the vertical blanking region, or vblank for
55 * short.
56 *
57 * For historical reference, the vertical blanking period was designed to
58 * give the electron gun (on CRTs) enough time to move back to the top of
59 * the screen to start scanning out the next frame. Similar for horizontal
60 * blanking periods. They were designed to give the electron gun enough
61 * time to move back to the other side of the screen to start scanning the
62 * next scanline.
63 *
64 * ::
65 *
66 *
67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68 * top of | |
69 * display | |
70 * | New frame |
71 * | |
72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
75 * | | frame as it
76 * | | travels down
77 * | | ("scan out")
78 * | Old frame |
79 * | |
80 * | |
81 * | |
82 * | | physical
83 * | | bottom of
84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89 * new frame
90 *
91 * "Physical top of display" is the reference point for the high-precision/
92 * corrected timestamp.
93 *
94 * On a lot of display hardware, programming needs to take effect during the
95 * vertical blanking period so that settings like gamma, the image buffer
96 * buffer to be scanned out, etc. can safely be changed without showing
97 * any visual artifacts on the screen. In some unforgiving hardware, some of
98 * this programming has to both start and end in the same vblank. To help
99 * with the timing of the hardware programming, an interrupt is usually
100 * available to notify the driver when it can start the updating of registers.
101 * The interrupt is in this context named the vblank interrupt.
102 *
103 * The vblank interrupt may be fired at different points depending on the
104 * hardware. Some hardware implementations will fire the interrupt when the
105 * new frame start, other implementations will fire the interrupt at different
106 * points in time.
107 *
108 * Vertical blanking plays a major role in graphics rendering. To achieve
109 * tear-free display, users must synchronize page flips and/or rendering to
110 * vertical blanking. The DRM API offers ioctls to perform page flips
111 * synchronized to vertical blanking and wait for vertical blanking.
112 *
113 * The DRM core handles most of the vertical blanking management logic, which
114 * involves filtering out spurious interrupts, keeping race-free blanking
115 * counters, coping with counter wrap-around and resets and keeping use counts.
116 * It relies on the driver to generate vertical blanking interrupts and
117 * optionally provide a hardware vertical blanking counter.
118 *
119 * Drivers must initialize the vertical blanking handling core with a call to
120 * drm_vblank_init(). Minimally, a driver needs to implement
121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123 * support.
124 *
125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
126 * themselves (for instance to handle page flipping operations). The DRM core
127 * maintains a vertical blanking use count to ensure that the interrupts are not
128 * disabled while a user still needs them. To increment the use count, drivers
129 * call drm_crtc_vblank_get() and release the vblank reference again with
130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131 * guaranteed to be enabled.
132 *
133 * On many hardware disabling the vblank interrupt cannot be done in a race-free
134 * manner, see &drm_driver.vblank_disable_immediate and
135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
136 * vblanks after a timer has expired, which can be configured through the
137 * ``vblankoffdelay`` module parameter.
138 *
139 * Drivers for hardware without support for vertical-blanking interrupts
140 * must not call drm_vblank_init(). For such drivers, atomic helpers will
141 * automatically generate fake vblank events as part of the display update.
142 * This functionality also can be controlled by the driver by enabling and
143 * disabling struct drm_crtc_state.no_vblank.
144 */
145
146/* Retry timestamp calculation up to 3 times to satisfy
147 * drm_timestamp_precision before giving up.
148 */
149#define DRM_TIMESTAMP_MAXRETRIES 3
150
151/* Threshold in nanoseconds for detection of redundant
152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153 */
154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
155
156static bool
157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
158 ktime_t *tvblank, bool in_vblank_irq);
159
160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
161
162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
163
164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168
169static void store_vblank(struct drm_device *dev, unsigned int pipe,
170 u32 vblank_count_inc,
171 ktime_t t_vblank, u32 last)
172{
173 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174
175 assert_spin_locked(&dev->vblank_time_lock);
176
177 vblank->last = last;
178
179 write_seqlock(sl: &vblank->seqlock);
180 vblank->time = t_vblank;
181 atomic64_add(i: vblank_count_inc, v: &vblank->count);
182 write_sequnlock(sl: &vblank->seqlock);
183}
184
185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186{
187 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188
189 return vblank->max_vblank_count ?: dev->max_vblank_count;
190}
191
192/*
193 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
194 * if there is no usable hardware frame counter available.
195 */
196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197{
198 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
199 return 0;
200}
201
202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203{
204 if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
205 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
206
207 if (drm_WARN_ON(dev, !crtc))
208 return 0;
209
210 if (crtc->funcs->get_vblank_counter)
211 return crtc->funcs->get_vblank_counter(crtc);
212 }
213#ifdef CONFIG_DRM_LEGACY
214 else if (dev->driver->get_vblank_counter) {
215 return dev->driver->get_vblank_counter(dev, pipe);
216 }
217#endif
218
219 return drm_vblank_no_hw_counter(dev, pipe);
220}
221
222/*
223 * Reset the stored timestamp for the current vblank count to correspond
224 * to the last vblank occurred.
225 *
226 * Only to be called from drm_crtc_vblank_on().
227 *
228 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
229 * device vblank fields.
230 */
231static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
232{
233 u32 cur_vblank;
234 bool rc;
235 ktime_t t_vblank;
236 int count = DRM_TIMESTAMP_MAXRETRIES;
237
238 spin_lock(lock: &dev->vblank_time_lock);
239
240 /*
241 * sample the current counter to avoid random jumps
242 * when drm_vblank_enable() applies the diff
243 */
244 do {
245 cur_vblank = __get_vblank_counter(dev, pipe);
246 rc = drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq: false);
247 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
248
249 /*
250 * Only reinitialize corresponding vblank timestamp if high-precision query
251 * available and didn't fail. Otherwise reinitialize delayed at next vblank
252 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
253 */
254 if (!rc)
255 t_vblank = 0;
256
257 /*
258 * +1 to make sure user will never see the same
259 * vblank counter value before and after a modeset
260 */
261 store_vblank(dev, pipe, vblank_count_inc: 1, t_vblank, last: cur_vblank);
262
263 spin_unlock(lock: &dev->vblank_time_lock);
264}
265
266/*
267 * Call back into the driver to update the appropriate vblank counter
268 * (specified by @pipe). Deal with wraparound, if it occurred, and
269 * update the last read value so we can deal with wraparound on the next
270 * call if necessary.
271 *
272 * Only necessary when going from off->on, to account for frames we
273 * didn't get an interrupt for.
274 *
275 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
276 * device vblank fields.
277 */
278static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
279 bool in_vblank_irq)
280{
281 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
282 u32 cur_vblank, diff;
283 bool rc;
284 ktime_t t_vblank;
285 int count = DRM_TIMESTAMP_MAXRETRIES;
286 int framedur_ns = vblank->framedur_ns;
287 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
288
289 /*
290 * Interrupts were disabled prior to this call, so deal with counter
291 * wrap if needed.
292 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
293 * here if the register is small or we had vblank interrupts off for
294 * a long time.
295 *
296 * We repeat the hardware vblank counter & timestamp query until
297 * we get consistent results. This to prevent races between gpu
298 * updating its hardware counter while we are retrieving the
299 * corresponding vblank timestamp.
300 */
301 do {
302 cur_vblank = __get_vblank_counter(dev, pipe);
303 rc = drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq);
304 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
305
306 if (max_vblank_count) {
307 /* trust the hw counter when it's around */
308 diff = (cur_vblank - vblank->last) & max_vblank_count;
309 } else if (rc && framedur_ns) {
310 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
311
312 /*
313 * Figure out how many vblanks we've missed based
314 * on the difference in the timestamps and the
315 * frame/field duration.
316 */
317
318 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
319 " diff_ns = %lld, framedur_ns = %d)\n",
320 pipe, (long long)diff_ns, framedur_ns);
321
322 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
323
324 if (diff == 0 && in_vblank_irq)
325 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
326 pipe);
327 } else {
328 /* some kind of default for drivers w/o accurate vbl timestamping */
329 diff = in_vblank_irq ? 1 : 0;
330 }
331
332 /*
333 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
334 * interval? If so then vblank irqs keep running and it will likely
335 * happen that the hardware vblank counter is not trustworthy as it
336 * might reset at some point in that interval and vblank timestamps
337 * are not trustworthy either in that interval. Iow. this can result
338 * in a bogus diff >> 1 which must be avoided as it would cause
339 * random large forward jumps of the software vblank counter.
340 */
341 if (diff > 1 && (vblank->inmodeset & 0x2)) {
342 drm_dbg_vbl(dev,
343 "clamping vblank bump to 1 on crtc %u: diffr=%u"
344 " due to pre-modeset.\n", pipe, diff);
345 diff = 1;
346 }
347
348 drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
349 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
350 pipe, (unsigned long long)atomic64_read(&vblank->count),
351 diff, cur_vblank, vblank->last);
352
353 if (diff == 0) {
354 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
355 return;
356 }
357
358 /*
359 * Only reinitialize corresponding vblank timestamp if high-precision query
360 * available and didn't fail, or we were called from the vblank interrupt.
361 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
362 * for now, to mark the vblanktimestamp as invalid.
363 */
364 if (!rc && !in_vblank_irq)
365 t_vblank = 0;
366
367 store_vblank(dev, pipe, vblank_count_inc: diff, t_vblank, last: cur_vblank);
368}
369
370u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
371{
372 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
373 u64 count;
374
375 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
376 return 0;
377
378 count = atomic64_read(v: &vblank->count);
379
380 /*
381 * This read barrier corresponds to the implicit write barrier of the
382 * write seqlock in store_vblank(). Note that this is the only place
383 * where we need an explicit barrier, since all other access goes
384 * through drm_vblank_count_and_time(), which already has the required
385 * read barrier curtesy of the read seqlock.
386 */
387 smp_rmb();
388
389 return count;
390}
391
392/**
393 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
394 * @crtc: which counter to retrieve
395 *
396 * This function is similar to drm_crtc_vblank_count() but this function
397 * interpolates to handle a race with vblank interrupts using the high precision
398 * timestamping support.
399 *
400 * This is mostly useful for hardware that can obtain the scanout position, but
401 * doesn't have a hardware frame counter.
402 */
403u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
404{
405 struct drm_device *dev = crtc->dev;
406 unsigned int pipe = drm_crtc_index(crtc);
407 u64 vblank;
408 unsigned long flags;
409
410 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
411 !crtc->funcs->get_vblank_timestamp,
412 "This function requires support for accurate vblank timestamps.");
413
414 spin_lock_irqsave(&dev->vblank_time_lock, flags);
415
416 drm_update_vblank_count(dev, pipe, in_vblank_irq: false);
417 vblank = drm_vblank_count(dev, pipe);
418
419 spin_unlock_irqrestore(lock: &dev->vblank_time_lock, flags);
420
421 return vblank;
422}
423EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
424
425static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
426{
427 if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
428 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
429
430 if (drm_WARN_ON(dev, !crtc))
431 return;
432
433 if (crtc->funcs->disable_vblank)
434 crtc->funcs->disable_vblank(crtc);
435 }
436#ifdef CONFIG_DRM_LEGACY
437 else {
438 dev->driver->disable_vblank(dev, pipe);
439 }
440#endif
441}
442
443/*
444 * Disable vblank irq's on crtc, make sure that last vblank count
445 * of hardware and corresponding consistent software vblank counter
446 * are preserved, even if there are any spurious vblank irq's after
447 * disable.
448 */
449void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
450{
451 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
452 unsigned long irqflags;
453
454 assert_spin_locked(&dev->vbl_lock);
455
456 /* Prevent vblank irq processing while disabling vblank irqs,
457 * so no updates of timestamps or count can happen after we've
458 * disabled. Needed to prevent races in case of delayed irq's.
459 */
460 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
461
462 /*
463 * Update vblank count and disable vblank interrupts only if the
464 * interrupts were enabled. This avoids calling the ->disable_vblank()
465 * operation in atomic context with the hardware potentially runtime
466 * suspended.
467 */
468 if (!vblank->enabled)
469 goto out;
470
471 /*
472 * Update the count and timestamp to maintain the
473 * appearance that the counter has been ticking all along until
474 * this time. This makes the count account for the entire time
475 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
476 */
477 drm_update_vblank_count(dev, pipe, in_vblank_irq: false);
478 __disable_vblank(dev, pipe);
479 vblank->enabled = false;
480
481out:
482 spin_unlock_irqrestore(lock: &dev->vblank_time_lock, flags: irqflags);
483}
484
485static void vblank_disable_fn(struct timer_list *t)
486{
487 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
488 struct drm_device *dev = vblank->dev;
489 unsigned int pipe = vblank->pipe;
490 unsigned long irqflags;
491
492 spin_lock_irqsave(&dev->vbl_lock, irqflags);
493 if (atomic_read(v: &vblank->refcount) == 0 && vblank->enabled) {
494 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
495 drm_vblank_disable_and_save(dev, pipe);
496 }
497 spin_unlock_irqrestore(lock: &dev->vbl_lock, flags: irqflags);
498}
499
500static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
501{
502 struct drm_vblank_crtc *vblank = ptr;
503
504 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
505 drm_core_check_feature(dev, DRIVER_MODESET));
506
507 drm_vblank_destroy_worker(vblank);
508 del_timer_sync(timer: &vblank->disable_timer);
509}
510
511/**
512 * drm_vblank_init - initialize vblank support
513 * @dev: DRM device
514 * @num_crtcs: number of CRTCs supported by @dev
515 *
516 * This function initializes vblank support for @num_crtcs display pipelines.
517 * Cleanup is handled automatically through a cleanup function added with
518 * drmm_add_action_or_reset().
519 *
520 * Returns:
521 * Zero on success or a negative error code on failure.
522 */
523int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
524{
525 int ret;
526 unsigned int i;
527
528 spin_lock_init(&dev->vbl_lock);
529 spin_lock_init(&dev->vblank_time_lock);
530
531 dev->vblank = drmm_kcalloc(dev, n: num_crtcs, size: sizeof(*dev->vblank), GFP_KERNEL);
532 if (!dev->vblank)
533 return -ENOMEM;
534
535 dev->num_crtcs = num_crtcs;
536
537 for (i = 0; i < num_crtcs; i++) {
538 struct drm_vblank_crtc *vblank = &dev->vblank[i];
539
540 vblank->dev = dev;
541 vblank->pipe = i;
542 init_waitqueue_head(&vblank->queue);
543 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
544 seqlock_init(&vblank->seqlock);
545
546 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
547 vblank);
548 if (ret)
549 return ret;
550
551 ret = drm_vblank_worker_init(vblank);
552 if (ret)
553 return ret;
554 }
555
556 return 0;
557}
558EXPORT_SYMBOL(drm_vblank_init);
559
560/**
561 * drm_dev_has_vblank - test if vblanking has been initialized for
562 * a device
563 * @dev: the device
564 *
565 * Drivers may call this function to test if vblank support is
566 * initialized for a device. For most hardware this means that vblanking
567 * can also be enabled.
568 *
569 * Atomic helpers use this function to initialize
570 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
571 *
572 * Returns:
573 * True if vblanking has been initialized for the given device, false
574 * otherwise.
575 */
576bool drm_dev_has_vblank(const struct drm_device *dev)
577{
578 return dev->num_crtcs != 0;
579}
580EXPORT_SYMBOL(drm_dev_has_vblank);
581
582/**
583 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
584 * @crtc: which CRTC's vblank waitqueue to retrieve
585 *
586 * This function returns a pointer to the vblank waitqueue for the CRTC.
587 * Drivers can use this to implement vblank waits using wait_event() and related
588 * functions.
589 */
590wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
591{
592 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
593}
594EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
595
596
597/**
598 * drm_calc_timestamping_constants - calculate vblank timestamp constants
599 * @crtc: drm_crtc whose timestamp constants should be updated.
600 * @mode: display mode containing the scanout timings
601 *
602 * Calculate and store various constants which are later needed by vblank and
603 * swap-completion timestamping, e.g, by
604 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
605 * CRTC's true scanout timing, so they take things like panel scaling or
606 * other adjustments into account.
607 */
608void drm_calc_timestamping_constants(struct drm_crtc *crtc,
609 const struct drm_display_mode *mode)
610{
611 struct drm_device *dev = crtc->dev;
612 unsigned int pipe = drm_crtc_index(crtc);
613 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
614 int linedur_ns = 0, framedur_ns = 0;
615 int dotclock = mode->crtc_clock;
616
617 if (!drm_dev_has_vblank(dev))
618 return;
619
620 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
621 return;
622
623 /* Valid dotclock? */
624 if (dotclock > 0) {
625 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
626
627 /*
628 * Convert scanline length in pixels and video
629 * dot clock to line duration and frame duration
630 * in nanoseconds:
631 */
632 linedur_ns = div_u64(dividend: (u64) mode->crtc_htotal * 1000000, divisor: dotclock);
633 framedur_ns = div_u64(dividend: (u64) frame_size * 1000000, divisor: dotclock);
634
635 /*
636 * Fields of interlaced scanout modes are only half a frame duration.
637 */
638 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
639 framedur_ns /= 2;
640 } else {
641 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
642 crtc->base.id);
643 }
644
645 vblank->linedur_ns = linedur_ns;
646 vblank->framedur_ns = framedur_ns;
647 drm_mode_copy(dst: &vblank->hwmode, src: mode);
648
649 drm_dbg_core(dev,
650 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
651 crtc->base.id, mode->crtc_htotal,
652 mode->crtc_vtotal, mode->crtc_vdisplay);
653 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
654 crtc->base.id, dotclock, framedur_ns, linedur_ns);
655}
656EXPORT_SYMBOL(drm_calc_timestamping_constants);
657
658/**
659 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
660 * timestamp helper
661 * @crtc: CRTC whose vblank timestamp to retrieve
662 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
663 * On return contains true maximum error of timestamp
664 * @vblank_time: Pointer to time which should receive the timestamp
665 * @in_vblank_irq:
666 * True when called from drm_crtc_handle_vblank(). Some drivers
667 * need to apply some workarounds for gpu-specific vblank irq quirks
668 * if flag is set.
669 * @get_scanout_position:
670 * Callback function to retrieve the scanout position. See
671 * @struct drm_crtc_helper_funcs.get_scanout_position.
672 *
673 * Implements calculation of exact vblank timestamps from given drm_display_mode
674 * timings and current video scanout position of a CRTC.
675 *
676 * The current implementation only handles standard video modes. For double scan
677 * and interlaced modes the driver is supposed to adjust the hardware mode
678 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
679 * match the scanout position reported.
680 *
681 * Note that atomic drivers must call drm_calc_timestamping_constants() before
682 * enabling a CRTC. The atomic helpers already take care of that in
683 * drm_atomic_helper_calc_timestamping_constants().
684 *
685 * Returns:
686 *
687 * Returns true on success, and false on failure, i.e. when no accurate
688 * timestamp could be acquired.
689 */
690bool
691drm_crtc_vblank_helper_get_vblank_timestamp_internal(
692 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
693 bool in_vblank_irq,
694 drm_vblank_get_scanout_position_func get_scanout_position)
695{
696 struct drm_device *dev = crtc->dev;
697 unsigned int pipe = crtc->index;
698 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
699 struct timespec64 ts_etime, ts_vblank_time;
700 ktime_t stime, etime;
701 bool vbl_status;
702 const struct drm_display_mode *mode;
703 int vpos, hpos, i;
704 int delta_ns, duration_ns;
705
706 if (pipe >= dev->num_crtcs) {
707 drm_err(dev, "Invalid crtc %u\n", pipe);
708 return false;
709 }
710
711 /* Scanout position query not supported? Should not happen. */
712 if (!get_scanout_position) {
713 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
714 return false;
715 }
716
717 if (drm_drv_uses_atomic_modeset(dev))
718 mode = &vblank->hwmode;
719 else
720 mode = &crtc->hwmode;
721
722 /* If mode timing undefined, just return as no-op:
723 * Happens during initial modesetting of a crtc.
724 */
725 if (mode->crtc_clock == 0) {
726 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
727 pipe);
728 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
729 return false;
730 }
731
732 /* Get current scanout position with system timestamp.
733 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
734 * if single query takes longer than max_error nanoseconds.
735 *
736 * This guarantees a tight bound on maximum error if
737 * code gets preempted or delayed for some reason.
738 */
739 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
740 /*
741 * Get vertical and horizontal scanout position vpos, hpos,
742 * and bounding timestamps stime, etime, pre/post query.
743 */
744 vbl_status = get_scanout_position(crtc, in_vblank_irq,
745 &vpos, &hpos,
746 &stime, &etime,
747 mode);
748
749 /* Return as no-op if scanout query unsupported or failed. */
750 if (!vbl_status) {
751 drm_dbg_core(dev,
752 "crtc %u : scanoutpos query failed.\n",
753 pipe);
754 return false;
755 }
756
757 /* Compute uncertainty in timestamp of scanout position query. */
758 duration_ns = ktime_to_ns(kt: etime) - ktime_to_ns(kt: stime);
759
760 /* Accept result with < max_error nsecs timing uncertainty. */
761 if (duration_ns <= *max_error)
762 break;
763 }
764
765 /* Noisy system timing? */
766 if (i == DRM_TIMESTAMP_MAXRETRIES) {
767 drm_dbg_core(dev,
768 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
769 pipe, duration_ns / 1000, *max_error / 1000, i);
770 }
771
772 /* Return upper bound of timestamp precision error. */
773 *max_error = duration_ns;
774
775 /* Convert scanout position into elapsed time at raw_time query
776 * since start of scanout at first display scanline. delta_ns
777 * can be negative if start of scanout hasn't happened yet.
778 */
779 delta_ns = div_s64(dividend: 1000000LL * (vpos * mode->crtc_htotal + hpos),
780 divisor: mode->crtc_clock);
781
782 /* Subtract time delta from raw timestamp to get final
783 * vblank_time timestamp for end of vblank.
784 */
785 *vblank_time = ktime_sub_ns(etime, delta_ns);
786
787 if (!drm_debug_enabled(DRM_UT_VBL))
788 return true;
789
790 ts_etime = ktime_to_timespec64(etime);
791 ts_vblank_time = ktime_to_timespec64(*vblank_time);
792
793 drm_dbg_vbl(dev,
794 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
795 pipe, hpos, vpos,
796 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
797 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
798 duration_ns / 1000, i);
799
800 return true;
801}
802EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
803
804/**
805 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
806 * helper
807 * @crtc: CRTC whose vblank timestamp to retrieve
808 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
809 * On return contains true maximum error of timestamp
810 * @vblank_time: Pointer to time which should receive the timestamp
811 * @in_vblank_irq:
812 * True when called from drm_crtc_handle_vblank(). Some drivers
813 * need to apply some workarounds for gpu-specific vblank irq quirks
814 * if flag is set.
815 *
816 * Implements calculation of exact vblank timestamps from given drm_display_mode
817 * timings and current video scanout position of a CRTC. This can be directly
818 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
819 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
820 *
821 * The current implementation only handles standard video modes. For double scan
822 * and interlaced modes the driver is supposed to adjust the hardware mode
823 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
824 * match the scanout position reported.
825 *
826 * Note that atomic drivers must call drm_calc_timestamping_constants() before
827 * enabling a CRTC. The atomic helpers already take care of that in
828 * drm_atomic_helper_calc_timestamping_constants().
829 *
830 * Returns:
831 *
832 * Returns true on success, and false on failure, i.e. when no accurate
833 * timestamp could be acquired.
834 */
835bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
836 int *max_error,
837 ktime_t *vblank_time,
838 bool in_vblank_irq)
839{
840 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
841 crtc, max_error, vblank_time, in_vblank_irq,
842 crtc->helper_private->get_scanout_position);
843}
844EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
845
846/**
847 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
848 * recent vblank interval
849 * @crtc: CRTC whose vblank timestamp to retrieve
850 * @tvblank: Pointer to target time which should receive the timestamp
851 * @in_vblank_irq:
852 * True when called from drm_crtc_handle_vblank(). Some drivers
853 * need to apply some workarounds for gpu-specific vblank irq quirks
854 * if flag is set.
855 *
856 * Fetches the system timestamp corresponding to the time of the most recent
857 * vblank interval on specified CRTC. May call into kms-driver to
858 * compute the timestamp with a high-precision GPU specific method.
859 *
860 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
861 * call, i.e., it isn't very precisely locked to the true vblank.
862 *
863 * Returns:
864 * True if timestamp is considered to be very precise, false otherwise.
865 */
866static bool
867drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
868 bool in_vblank_irq)
869{
870 bool ret = false;
871
872 /* Define requested maximum error on timestamps (nanoseconds). */
873 int max_error = (int) drm_timestamp_precision * 1000;
874
875 /* Query driver if possible and precision timestamping enabled. */
876 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
877 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
878 tvblank, in_vblank_irq);
879 }
880
881 /* GPU high precision timestamp query unsupported or failed.
882 * Return current monotonic/gettimeofday timestamp as best estimate.
883 */
884 if (!ret)
885 *tvblank = ktime_get();
886
887 return ret;
888}
889
890static bool
891drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
892 ktime_t *tvblank, bool in_vblank_irq)
893{
894 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
895
896 return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
897}
898
899/**
900 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
901 * @crtc: which counter to retrieve
902 *
903 * Fetches the "cooked" vblank count value that represents the number of
904 * vblank events since the system was booted, including lost events due to
905 * modesetting activity. Note that this timer isn't correct against a racing
906 * vblank interrupt (since it only reports the software vblank counter), see
907 * drm_crtc_accurate_vblank_count() for such use-cases.
908 *
909 * Note that for a given vblank counter value drm_crtc_handle_vblank()
910 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
911 * provide a barrier: Any writes done before calling
912 * drm_crtc_handle_vblank() will be visible to callers of the later
913 * functions, if the vblank count is the same or a later one.
914 *
915 * See also &drm_vblank_crtc.count.
916 *
917 * Returns:
918 * The software vblank counter.
919 */
920u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
921{
922 return drm_vblank_count(dev: crtc->dev, pipe: drm_crtc_index(crtc));
923}
924EXPORT_SYMBOL(drm_crtc_vblank_count);
925
926/**
927 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
928 * system timestamp corresponding to that vblank counter value.
929 * @dev: DRM device
930 * @pipe: index of CRTC whose counter to retrieve
931 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
932 *
933 * Fetches the "cooked" vblank count value that represents the number of
934 * vblank events since the system was booted, including lost events due to
935 * modesetting activity. Returns corresponding system timestamp of the time
936 * of the vblank interval that corresponds to the current vblank counter value.
937 *
938 * This is the legacy version of drm_crtc_vblank_count_and_time().
939 */
940static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
941 ktime_t *vblanktime)
942{
943 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
944 u64 vblank_count;
945 unsigned int seq;
946
947 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
948 *vblanktime = 0;
949 return 0;
950 }
951
952 do {
953 seq = read_seqbegin(sl: &vblank->seqlock);
954 vblank_count = atomic64_read(v: &vblank->count);
955 *vblanktime = vblank->time;
956 } while (read_seqretry(sl: &vblank->seqlock, start: seq));
957
958 return vblank_count;
959}
960
961/**
962 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
963 * and the system timestamp corresponding to that vblank counter value
964 * @crtc: which counter to retrieve
965 * @vblanktime: Pointer to time to receive the vblank timestamp.
966 *
967 * Fetches the "cooked" vblank count value that represents the number of
968 * vblank events since the system was booted, including lost events due to
969 * modesetting activity. Returns corresponding system timestamp of the time
970 * of the vblank interval that corresponds to the current vblank counter value.
971 *
972 * Note that for a given vblank counter value drm_crtc_handle_vblank()
973 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
974 * provide a barrier: Any writes done before calling
975 * drm_crtc_handle_vblank() will be visible to callers of the later
976 * functions, if the vblank count is the same or a later one.
977 *
978 * See also &drm_vblank_crtc.count.
979 */
980u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
981 ktime_t *vblanktime)
982{
983 return drm_vblank_count_and_time(dev: crtc->dev, pipe: drm_crtc_index(crtc),
984 vblanktime);
985}
986EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
987
988/**
989 * drm_crtc_next_vblank_start - calculate the time of the next vblank
990 * @crtc: the crtc for which to calculate next vblank time
991 * @vblanktime: pointer to time to receive the next vblank timestamp.
992 *
993 * Calculate the expected time of the start of the next vblank period,
994 * based on time of previous vblank and frame duration
995 */
996int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
997{
998 unsigned int pipe = drm_crtc_index(crtc);
999 struct drm_vblank_crtc *vblank;
1000 struct drm_display_mode *mode;
1001 u64 vblank_start;
1002
1003 if (!drm_dev_has_vblank(crtc->dev))
1004 return -EINVAL;
1005
1006 vblank = &crtc->dev->vblank[pipe];
1007 mode = &vblank->hwmode;
1008
1009 if (!vblank->framedur_ns || !vblank->linedur_ns)
1010 return -EINVAL;
1011
1012 if (!drm_crtc_get_last_vbltimestamp(crtc, tvblank: vblanktime, in_vblank_irq: false))
1013 return -EINVAL;
1014
1015 vblank_start = DIV_ROUND_DOWN_ULL(
1016 (u64)vblank->framedur_ns * mode->crtc_vblank_start,
1017 mode->crtc_vtotal);
1018 *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1019
1020 return 0;
1021}
1022EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1023
1024static void send_vblank_event(struct drm_device *dev,
1025 struct drm_pending_vblank_event *e,
1026 u64 seq, ktime_t now)
1027{
1028 struct timespec64 tv;
1029
1030 switch (e->event.base.type) {
1031 case DRM_EVENT_VBLANK:
1032 case DRM_EVENT_FLIP_COMPLETE:
1033 tv = ktime_to_timespec64(now);
1034 e->event.vbl.sequence = seq;
1035 /*
1036 * e->event is a user space structure, with hardcoded unsigned
1037 * 32-bit seconds/microseconds. This is safe as we always use
1038 * monotonic timestamps since linux-4.15
1039 */
1040 e->event.vbl.tv_sec = tv.tv_sec;
1041 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1042 break;
1043 case DRM_EVENT_CRTC_SEQUENCE:
1044 if (seq)
1045 e->event.seq.sequence = seq;
1046 e->event.seq.time_ns = ktime_to_ns(kt: now);
1047 break;
1048 }
1049 trace_drm_vblank_event_delivered(file: e->base.file_priv, crtc: e->pipe, seq);
1050 /*
1051 * Use the same timestamp for any associated fence signal to avoid
1052 * mismatch in timestamps for vsync & fence events triggered by the
1053 * same HW event. Frameworks like SurfaceFlinger in Android expects the
1054 * retire-fence timestamp to match exactly with HW vsync as it uses it
1055 * for its software vsync modeling.
1056 */
1057 drm_send_event_timestamp_locked(dev, e: &e->base, timestamp: now);
1058}
1059
1060/**
1061 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1062 * @crtc: the source CRTC of the vblank event
1063 * @e: the event to send
1064 *
1065 * A lot of drivers need to generate vblank events for the very next vblank
1066 * interrupt. For example when the page flip interrupt happens when the page
1067 * flip gets armed, but not when it actually executes within the next vblank
1068 * period. This helper function implements exactly the required vblank arming
1069 * behaviour.
1070 *
1071 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1072 * atomic commit must ensure that the next vblank happens at exactly the same
1073 * time as the atomic commit is committed to the hardware. This function itself
1074 * does **not** protect against the next vblank interrupt racing with either this
1075 * function call or the atomic commit operation. A possible sequence could be:
1076 *
1077 * 1. Driver commits new hardware state into vblank-synchronized registers.
1078 * 2. A vblank happens, committing the hardware state. Also the corresponding
1079 * vblank interrupt is fired off and fully processed by the interrupt
1080 * handler.
1081 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1082 * 4. The event is only send out for the next vblank, which is wrong.
1083 *
1084 * An equivalent race can happen when the driver calls
1085 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1086 *
1087 * The only way to make this work safely is to prevent the vblank from firing
1088 * (and the hardware from committing anything else) until the entire atomic
1089 * commit sequence has run to completion. If the hardware does not have such a
1090 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1091 * Instead drivers need to manually send out the event from their interrupt
1092 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1093 * possible race with the hardware committing the atomic update.
1094 *
1095 * Caller must hold a vblank reference for the event @e acquired by a
1096 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1097 */
1098void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1099 struct drm_pending_vblank_event *e)
1100{
1101 struct drm_device *dev = crtc->dev;
1102 unsigned int pipe = drm_crtc_index(crtc);
1103
1104 assert_spin_locked(&dev->event_lock);
1105
1106 e->pipe = pipe;
1107 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1108 list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
1109}
1110EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1111
1112/**
1113 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1114 * @crtc: the source CRTC of the vblank event
1115 * @e: the event to send
1116 *
1117 * Updates sequence # and timestamp on event for the most recently processed
1118 * vblank, and sends it to userspace. Caller must hold event lock.
1119 *
1120 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1121 * situation, especially to send out events for atomic commit operations.
1122 */
1123void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1124 struct drm_pending_vblank_event *e)
1125{
1126 struct drm_device *dev = crtc->dev;
1127 u64 seq;
1128 unsigned int pipe = drm_crtc_index(crtc);
1129 ktime_t now;
1130
1131 if (drm_dev_has_vblank(dev)) {
1132 seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1133 } else {
1134 seq = 0;
1135
1136 now = ktime_get();
1137 }
1138 e->pipe = pipe;
1139 send_vblank_event(dev, e, seq, now);
1140}
1141EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1142
1143static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1144{
1145 if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1146 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1147
1148 if (drm_WARN_ON(dev, !crtc))
1149 return 0;
1150
1151 if (crtc->funcs->enable_vblank)
1152 return crtc->funcs->enable_vblank(crtc);
1153 }
1154#ifdef CONFIG_DRM_LEGACY
1155 else if (dev->driver->enable_vblank) {
1156 return dev->driver->enable_vblank(dev, pipe);
1157 }
1158#endif
1159
1160 return -EINVAL;
1161}
1162
1163static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1164{
1165 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1166 int ret = 0;
1167
1168 assert_spin_locked(&dev->vbl_lock);
1169
1170 spin_lock(lock: &dev->vblank_time_lock);
1171
1172 if (!vblank->enabled) {
1173 /*
1174 * Enable vblank irqs under vblank_time_lock protection.
1175 * All vblank count & timestamp updates are held off
1176 * until we are done reinitializing master counter and
1177 * timestamps. Filtercode in drm_handle_vblank() will
1178 * prevent double-accounting of same vblank interval.
1179 */
1180 ret = __enable_vblank(dev, pipe);
1181 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1182 pipe, ret);
1183 if (ret) {
1184 atomic_dec(v: &vblank->refcount);
1185 } else {
1186 drm_update_vblank_count(dev, pipe, in_vblank_irq: 0);
1187 /* drm_update_vblank_count() includes a wmb so we just
1188 * need to ensure that the compiler emits the write
1189 * to mark the vblank as enabled after the call
1190 * to drm_update_vblank_count().
1191 */
1192 WRITE_ONCE(vblank->enabled, true);
1193 }
1194 }
1195
1196 spin_unlock(lock: &dev->vblank_time_lock);
1197
1198 return ret;
1199}
1200
1201int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1202{
1203 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1204 unsigned long irqflags;
1205 int ret = 0;
1206
1207 if (!drm_dev_has_vblank(dev))
1208 return -EINVAL;
1209
1210 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1211 return -EINVAL;
1212
1213 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1214 /* Going from 0->1 means we have to enable interrupts again */
1215 if (atomic_add_return(i: 1, v: &vblank->refcount) == 1) {
1216 ret = drm_vblank_enable(dev, pipe);
1217 } else {
1218 if (!vblank->enabled) {
1219 atomic_dec(v: &vblank->refcount);
1220 ret = -EINVAL;
1221 }
1222 }
1223 spin_unlock_irqrestore(lock: &dev->vbl_lock, flags: irqflags);
1224
1225 return ret;
1226}
1227
1228/**
1229 * drm_crtc_vblank_get - get a reference count on vblank events
1230 * @crtc: which CRTC to own
1231 *
1232 * Acquire a reference count on vblank events to avoid having them disabled
1233 * while in use.
1234 *
1235 * Returns:
1236 * Zero on success or a negative error code on failure.
1237 */
1238int drm_crtc_vblank_get(struct drm_crtc *crtc)
1239{
1240 return drm_vblank_get(dev: crtc->dev, pipe: drm_crtc_index(crtc));
1241}
1242EXPORT_SYMBOL(drm_crtc_vblank_get);
1243
1244void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1245{
1246 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247
1248 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1249 return;
1250
1251 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1252 return;
1253
1254 /* Last user schedules interrupt disable */
1255 if (atomic_dec_and_test(v: &vblank->refcount)) {
1256 if (drm_vblank_offdelay == 0)
1257 return;
1258 else if (drm_vblank_offdelay < 0)
1259 vblank_disable_fn(t: &vblank->disable_timer);
1260 else if (!dev->vblank_disable_immediate)
1261 mod_timer(timer: &vblank->disable_timer,
1262 expires: jiffies + ((drm_vblank_offdelay * HZ)/1000));
1263 }
1264}
1265
1266/**
1267 * drm_crtc_vblank_put - give up ownership of vblank events
1268 * @crtc: which counter to give up
1269 *
1270 * Release ownership of a given vblank counter, turning off interrupts
1271 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1272 */
1273void drm_crtc_vblank_put(struct drm_crtc *crtc)
1274{
1275 drm_vblank_put(dev: crtc->dev, pipe: drm_crtc_index(crtc));
1276}
1277EXPORT_SYMBOL(drm_crtc_vblank_put);
1278
1279/**
1280 * drm_wait_one_vblank - wait for one vblank
1281 * @dev: DRM device
1282 * @pipe: CRTC index
1283 *
1284 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1285 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1286 * due to lack of driver support or because the crtc is off.
1287 *
1288 * This is the legacy version of drm_crtc_wait_one_vblank().
1289 */
1290void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1291{
1292 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1293 int ret;
1294 u64 last;
1295
1296 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1297 return;
1298
1299 ret = drm_vblank_get(dev, pipe);
1300 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1301 pipe, ret))
1302 return;
1303
1304 last = drm_vblank_count(dev, pipe);
1305
1306 ret = wait_event_timeout(vblank->queue,
1307 last != drm_vblank_count(dev, pipe),
1308 msecs_to_jiffies(100));
1309
1310 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1311
1312 drm_vblank_put(dev, pipe);
1313}
1314EXPORT_SYMBOL(drm_wait_one_vblank);
1315
1316/**
1317 * drm_crtc_wait_one_vblank - wait for one vblank
1318 * @crtc: DRM crtc
1319 *
1320 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1321 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1322 * due to lack of driver support or because the crtc is off.
1323 */
1324void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1325{
1326 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1327}
1328EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1329
1330/**
1331 * drm_crtc_vblank_off - disable vblank events on a CRTC
1332 * @crtc: CRTC in question
1333 *
1334 * Drivers can use this function to shut down the vblank interrupt handling when
1335 * disabling a crtc. This function ensures that the latest vblank frame count is
1336 * stored so that drm_vblank_on can restore it again.
1337 *
1338 * Drivers must use this function when the hardware vblank counter can get
1339 * reset, e.g. when suspending or disabling the @crtc in general.
1340 */
1341void drm_crtc_vblank_off(struct drm_crtc *crtc)
1342{
1343 struct drm_device *dev = crtc->dev;
1344 unsigned int pipe = drm_crtc_index(crtc);
1345 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1346 struct drm_pending_vblank_event *e, *t;
1347 ktime_t now;
1348 u64 seq;
1349
1350 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1351 return;
1352
1353 /*
1354 * Grab event_lock early to prevent vblank work from being scheduled
1355 * while we're in the middle of shutting down vblank interrupts
1356 */
1357 spin_lock_irq(lock: &dev->event_lock);
1358
1359 spin_lock(lock: &dev->vbl_lock);
1360 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1361 pipe, vblank->enabled, vblank->inmodeset);
1362
1363 /* Avoid redundant vblank disables without previous
1364 * drm_crtc_vblank_on(). */
1365 if (drm_core_check_feature(dev, feature: DRIVER_ATOMIC) || !vblank->inmodeset)
1366 drm_vblank_disable_and_save(dev, pipe);
1367
1368 wake_up(&vblank->queue);
1369
1370 /*
1371 * Prevent subsequent drm_vblank_get() from re-enabling
1372 * the vblank interrupt by bumping the refcount.
1373 */
1374 if (!vblank->inmodeset) {
1375 atomic_inc(v: &vblank->refcount);
1376 vblank->inmodeset = 1;
1377 }
1378 spin_unlock(lock: &dev->vbl_lock);
1379
1380 /* Send any queued vblank events, lest the natives grow disquiet */
1381 seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1382
1383 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1384 if (e->pipe != pipe)
1385 continue;
1386 drm_dbg_core(dev, "Sending premature vblank event on disable: "
1387 "wanted %llu, current %llu\n",
1388 e->sequence, seq);
1389 list_del(entry: &e->base.link);
1390 drm_vblank_put(dev, pipe);
1391 send_vblank_event(dev, e, seq, now);
1392 }
1393
1394 /* Cancel any leftover pending vblank work */
1395 drm_vblank_cancel_pending_works(vblank);
1396
1397 spin_unlock_irq(lock: &dev->event_lock);
1398
1399 /* Will be reset by the modeset helpers when re-enabling the crtc by
1400 * calling drm_calc_timestamping_constants(). */
1401 vblank->hwmode.crtc_clock = 0;
1402
1403 /* Wait for any vblank work that's still executing to finish */
1404 drm_vblank_flush_worker(vblank);
1405}
1406EXPORT_SYMBOL(drm_crtc_vblank_off);
1407
1408/**
1409 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1410 * @crtc: CRTC in question
1411 *
1412 * Drivers can use this function to reset the vblank state to off at load time.
1413 * Drivers should use this together with the drm_crtc_vblank_off() and
1414 * drm_crtc_vblank_on() functions. The difference compared to
1415 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1416 * and hence doesn't need to call any driver hooks.
1417 *
1418 * This is useful for recovering driver state e.g. on driver load, or on resume.
1419 */
1420void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1421{
1422 struct drm_device *dev = crtc->dev;
1423 unsigned int pipe = drm_crtc_index(crtc);
1424 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1425
1426 spin_lock_irq(lock: &dev->vbl_lock);
1427 /*
1428 * Prevent subsequent drm_vblank_get() from enabling the vblank
1429 * interrupt by bumping the refcount.
1430 */
1431 if (!vblank->inmodeset) {
1432 atomic_inc(v: &vblank->refcount);
1433 vblank->inmodeset = 1;
1434 }
1435 spin_unlock_irq(lock: &dev->vbl_lock);
1436
1437 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1438 drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1439}
1440EXPORT_SYMBOL(drm_crtc_vblank_reset);
1441
1442/**
1443 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1444 * @crtc: CRTC in question
1445 * @max_vblank_count: max hardware vblank counter value
1446 *
1447 * Update the maximum hardware vblank counter value for @crtc
1448 * at runtime. Useful for hardware where the operation of the
1449 * hardware vblank counter depends on the currently active
1450 * display configuration.
1451 *
1452 * For example, if the hardware vblank counter does not work
1453 * when a specific connector is active the maximum can be set
1454 * to zero. And when that specific connector isn't active the
1455 * maximum can again be set to the appropriate non-zero value.
1456 *
1457 * If used, must be called before drm_vblank_on().
1458 */
1459void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1460 u32 max_vblank_count)
1461{
1462 struct drm_device *dev = crtc->dev;
1463 unsigned int pipe = drm_crtc_index(crtc);
1464 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1465
1466 drm_WARN_ON(dev, dev->max_vblank_count);
1467 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1468
1469 vblank->max_vblank_count = max_vblank_count;
1470}
1471EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1472
1473/**
1474 * drm_crtc_vblank_on - enable vblank events on a CRTC
1475 * @crtc: CRTC in question
1476 *
1477 * This functions restores the vblank interrupt state captured with
1478 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1479 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1480 * unbalanced and so can also be unconditionally called in driver load code to
1481 * reflect the current hardware state of the crtc.
1482 */
1483void drm_crtc_vblank_on(struct drm_crtc *crtc)
1484{
1485 struct drm_device *dev = crtc->dev;
1486 unsigned int pipe = drm_crtc_index(crtc);
1487 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1488
1489 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1490 return;
1491
1492 spin_lock_irq(lock: &dev->vbl_lock);
1493 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1494 pipe, vblank->enabled, vblank->inmodeset);
1495
1496 /* Drop our private "prevent drm_vblank_get" refcount */
1497 if (vblank->inmodeset) {
1498 atomic_dec(v: &vblank->refcount);
1499 vblank->inmodeset = 0;
1500 }
1501
1502 drm_reset_vblank_timestamp(dev, pipe);
1503
1504 /*
1505 * re-enable interrupts if there are users left, or the
1506 * user wishes vblank interrupts to be enabled all the time.
1507 */
1508 if (atomic_read(v: &vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1509 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1510 spin_unlock_irq(lock: &dev->vbl_lock);
1511}
1512EXPORT_SYMBOL(drm_crtc_vblank_on);
1513
1514static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1515{
1516 ktime_t t_vblank;
1517 struct drm_vblank_crtc *vblank;
1518 int framedur_ns;
1519 u64 diff_ns;
1520 u32 cur_vblank, diff = 1;
1521 int count = DRM_TIMESTAMP_MAXRETRIES;
1522 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1523
1524 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1525 return;
1526
1527 assert_spin_locked(&dev->vbl_lock);
1528 assert_spin_locked(&dev->vblank_time_lock);
1529
1530 vblank = &dev->vblank[pipe];
1531 drm_WARN_ONCE(dev,
1532 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1533 "Cannot compute missed vblanks without frame duration\n");
1534 framedur_ns = vblank->framedur_ns;
1535
1536 do {
1537 cur_vblank = __get_vblank_counter(dev, pipe);
1538 drm_get_last_vbltimestamp(dev, pipe, tvblank: &t_vblank, in_vblank_irq: false);
1539 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1540
1541 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1542 if (framedur_ns)
1543 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1544
1545
1546 drm_dbg_vbl(dev,
1547 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1548 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1549 vblank->last = (cur_vblank - diff) & max_vblank_count;
1550}
1551
1552/**
1553 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1554 * @crtc: CRTC in question
1555 *
1556 * Power manamement features can cause frame counter resets between vblank
1557 * disable and enable. Drivers can use this function in their
1558 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1559 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1560 * vblank counter.
1561 *
1562 * Note that drivers must have race-free high-precision timestamping support,
1563 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1564 * &drm_driver.vblank_disable_immediate must be set to indicate the
1565 * time-stamping functions are race-free against vblank hardware counter
1566 * increments.
1567 */
1568void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1569{
1570 WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
1571 WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
1572
1573 drm_vblank_restore(dev: crtc->dev, pipe: drm_crtc_index(crtc));
1574}
1575EXPORT_SYMBOL(drm_crtc_vblank_restore);
1576
1577static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1578 unsigned int pipe)
1579{
1580 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1581
1582 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1583 if (!drm_dev_has_vblank(dev))
1584 return;
1585
1586 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1587 return;
1588
1589 /*
1590 * To avoid all the problems that might happen if interrupts
1591 * were enabled/disabled around or between these calls, we just
1592 * have the kernel take a reference on the CRTC (just once though
1593 * to avoid corrupting the count if multiple, mismatch calls occur),
1594 * so that interrupts remain enabled in the interim.
1595 */
1596 if (!vblank->inmodeset) {
1597 vblank->inmodeset = 0x1;
1598 if (drm_vblank_get(dev, pipe) == 0)
1599 vblank->inmodeset |= 0x2;
1600 }
1601}
1602
1603static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1604 unsigned int pipe)
1605{
1606 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1607
1608 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1609 if (!drm_dev_has_vblank(dev))
1610 return;
1611
1612 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1613 return;
1614
1615 if (vblank->inmodeset) {
1616 spin_lock_irq(lock: &dev->vbl_lock);
1617 drm_reset_vblank_timestamp(dev, pipe);
1618 spin_unlock_irq(lock: &dev->vbl_lock);
1619
1620 if (vblank->inmodeset & 0x2)
1621 drm_vblank_put(dev, pipe);
1622
1623 vblank->inmodeset = 0;
1624 }
1625}
1626
1627int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1628 struct drm_file *file_priv)
1629{
1630 struct drm_modeset_ctl *modeset = data;
1631 unsigned int pipe;
1632
1633 /* If drm_vblank_init() hasn't been called yet, just no-op */
1634 if (!drm_dev_has_vblank(dev))
1635 return 0;
1636
1637 /* KMS drivers handle this internally */
1638 if (!drm_core_check_feature(dev, feature: DRIVER_LEGACY))
1639 return 0;
1640
1641 pipe = modeset->crtc;
1642 if (pipe >= dev->num_crtcs)
1643 return -EINVAL;
1644
1645 switch (modeset->cmd) {
1646 case _DRM_PRE_MODESET:
1647 drm_legacy_vblank_pre_modeset(dev, pipe);
1648 break;
1649 case _DRM_POST_MODESET:
1650 drm_legacy_vblank_post_modeset(dev, pipe);
1651 break;
1652 default:
1653 return -EINVAL;
1654 }
1655
1656 return 0;
1657}
1658
1659static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1660 u64 req_seq,
1661 union drm_wait_vblank *vblwait,
1662 struct drm_file *file_priv)
1663{
1664 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1665 struct drm_pending_vblank_event *e;
1666 ktime_t now;
1667 u64 seq;
1668 int ret;
1669
1670 e = kzalloc(size: sizeof(*e), GFP_KERNEL);
1671 if (e == NULL) {
1672 ret = -ENOMEM;
1673 goto err_put;
1674 }
1675
1676 e->pipe = pipe;
1677 e->event.base.type = DRM_EVENT_VBLANK;
1678 e->event.base.length = sizeof(e->event.vbl);
1679 e->event.vbl.user_data = vblwait->request.signal;
1680 e->event.vbl.crtc_id = 0;
1681 if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1682 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1683
1684 if (crtc)
1685 e->event.vbl.crtc_id = crtc->base.id;
1686 }
1687
1688 spin_lock_irq(lock: &dev->event_lock);
1689
1690 /*
1691 * drm_crtc_vblank_off() might have been called after we called
1692 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1693 * vblank disable, so no need for further locking. The reference from
1694 * drm_vblank_get() protects against vblank disable from another source.
1695 */
1696 if (!READ_ONCE(vblank->enabled)) {
1697 ret = -EINVAL;
1698 goto err_unlock;
1699 }
1700
1701 ret = drm_event_reserve_init_locked(dev, file_priv, p: &e->base,
1702 e: &e->event.base);
1703
1704 if (ret)
1705 goto err_unlock;
1706
1707 seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1708
1709 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1710 req_seq, seq, pipe);
1711
1712 trace_drm_vblank_event_queued(file: file_priv, crtc: pipe, seq: req_seq);
1713
1714 e->sequence = req_seq;
1715 if (drm_vblank_passed(seq, ref: req_seq)) {
1716 drm_vblank_put(dev, pipe);
1717 send_vblank_event(dev, e, seq, now);
1718 vblwait->reply.sequence = seq;
1719 } else {
1720 /* drm_handle_vblank_events will call drm_vblank_put */
1721 list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
1722 vblwait->reply.sequence = req_seq;
1723 }
1724
1725 spin_unlock_irq(lock: &dev->event_lock);
1726
1727 return 0;
1728
1729err_unlock:
1730 spin_unlock_irq(lock: &dev->event_lock);
1731 kfree(objp: e);
1732err_put:
1733 drm_vblank_put(dev, pipe);
1734 return ret;
1735}
1736
1737static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1738{
1739 if (vblwait->request.sequence)
1740 return false;
1741
1742 return _DRM_VBLANK_RELATIVE ==
1743 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1744 _DRM_VBLANK_EVENT |
1745 _DRM_VBLANK_NEXTONMISS));
1746}
1747
1748/*
1749 * Widen a 32-bit param to 64-bits.
1750 *
1751 * \param narrow 32-bit value (missing upper 32 bits)
1752 * \param near 64-bit value that should be 'close' to near
1753 *
1754 * This function returns a 64-bit value using the lower 32-bits from
1755 * 'narrow' and constructing the upper 32-bits so that the result is
1756 * as close as possible to 'near'.
1757 */
1758
1759static u64 widen_32_to_64(u32 narrow, u64 near)
1760{
1761 return near + (s32) (narrow - near);
1762}
1763
1764static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1765 struct drm_wait_vblank_reply *reply)
1766{
1767 ktime_t now;
1768 struct timespec64 ts;
1769
1770 /*
1771 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1772 * to store the seconds. This is safe as we always use monotonic
1773 * timestamps since linux-4.15.
1774 */
1775 reply->sequence = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1776 ts = ktime_to_timespec64(now);
1777 reply->tval_sec = (u32)ts.tv_sec;
1778 reply->tval_usec = ts.tv_nsec / 1000;
1779}
1780
1781static bool drm_wait_vblank_supported(struct drm_device *dev)
1782{
1783#if IS_ENABLED(CONFIG_DRM_LEGACY)
1784 if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY)))
1785 return dev->irq_enabled;
1786#endif
1787 return drm_dev_has_vblank(dev);
1788}
1789
1790int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1791 struct drm_file *file_priv)
1792{
1793 struct drm_crtc *crtc;
1794 struct drm_vblank_crtc *vblank;
1795 union drm_wait_vblank *vblwait = data;
1796 int ret;
1797 u64 req_seq, seq;
1798 unsigned int pipe_index;
1799 unsigned int flags, pipe, high_pipe;
1800
1801 if (!drm_wait_vblank_supported(dev))
1802 return -EOPNOTSUPP;
1803
1804 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1805 return -EINVAL;
1806
1807 if (vblwait->request.type &
1808 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1809 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1810 drm_dbg_core(dev,
1811 "Unsupported type value 0x%x, supported mask 0x%x\n",
1812 vblwait->request.type,
1813 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1814 _DRM_VBLANK_HIGH_CRTC_MASK));
1815 return -EINVAL;
1816 }
1817
1818 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1819 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1820 if (high_pipe)
1821 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1822 else
1823 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1824
1825 /* Convert lease-relative crtc index into global crtc index */
1826 if (drm_core_check_feature(dev, feature: DRIVER_MODESET)) {
1827 pipe = 0;
1828 drm_for_each_crtc(crtc, dev) {
1829 if (drm_lease_held(file_priv, id: crtc->base.id)) {
1830 if (pipe_index == 0)
1831 break;
1832 pipe_index--;
1833 }
1834 pipe++;
1835 }
1836 } else {
1837 pipe = pipe_index;
1838 }
1839
1840 if (pipe >= dev->num_crtcs)
1841 return -EINVAL;
1842
1843 vblank = &dev->vblank[pipe];
1844
1845 /* If the counter is currently enabled and accurate, short-circuit
1846 * queries to return the cached timestamp of the last vblank.
1847 */
1848 if (dev->vblank_disable_immediate &&
1849 drm_wait_vblank_is_query(vblwait) &&
1850 READ_ONCE(vblank->enabled)) {
1851 drm_wait_vblank_reply(dev, pipe, reply: &vblwait->reply);
1852 return 0;
1853 }
1854
1855 ret = drm_vblank_get(dev, pipe);
1856 if (ret) {
1857 drm_dbg_core(dev,
1858 "crtc %d failed to acquire vblank counter, %d\n",
1859 pipe, ret);
1860 return ret;
1861 }
1862 seq = drm_vblank_count(dev, pipe);
1863
1864 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1865 case _DRM_VBLANK_RELATIVE:
1866 req_seq = seq + vblwait->request.sequence;
1867 vblwait->request.sequence = req_seq;
1868 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1869 break;
1870 case _DRM_VBLANK_ABSOLUTE:
1871 req_seq = widen_32_to_64(narrow: vblwait->request.sequence, near: seq);
1872 break;
1873 default:
1874 ret = -EINVAL;
1875 goto done;
1876 }
1877
1878 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1879 drm_vblank_passed(seq, ref: req_seq)) {
1880 req_seq = seq + 1;
1881 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1882 vblwait->request.sequence = req_seq;
1883 }
1884
1885 if (flags & _DRM_VBLANK_EVENT) {
1886 /* must hold on to the vblank ref until the event fires
1887 * drm_vblank_put will be called asynchronously
1888 */
1889 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1890 }
1891
1892 if (req_seq != seq) {
1893 int wait;
1894
1895 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1896 req_seq, pipe);
1897 wait = wait_event_interruptible_timeout(vblank->queue,
1898 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1899 !READ_ONCE(vblank->enabled),
1900 msecs_to_jiffies(3000));
1901
1902 switch (wait) {
1903 case 0:
1904 /* timeout */
1905 ret = -EBUSY;
1906 break;
1907 case -ERESTARTSYS:
1908 /* interrupted by signal */
1909 ret = -EINTR;
1910 break;
1911 default:
1912 ret = 0;
1913 break;
1914 }
1915 }
1916
1917 if (ret != -EINTR) {
1918 drm_wait_vblank_reply(dev, pipe, reply: &vblwait->reply);
1919
1920 drm_dbg_core(dev, "crtc %d returning %u to client\n",
1921 pipe, vblwait->reply.sequence);
1922 } else {
1923 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1924 pipe);
1925 }
1926
1927done:
1928 drm_vblank_put(dev, pipe);
1929 return ret;
1930}
1931
1932static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1933{
1934 struct drm_crtc *crtc = drm_crtc_from_index(dev, idx: pipe);
1935 bool high_prec = false;
1936 struct drm_pending_vblank_event *e, *t;
1937 ktime_t now;
1938 u64 seq;
1939
1940 assert_spin_locked(&dev->event_lock);
1941
1942 seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
1943
1944 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1945 if (e->pipe != pipe)
1946 continue;
1947 if (!drm_vblank_passed(seq, ref: e->sequence))
1948 continue;
1949
1950 drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1951 e->sequence, seq);
1952
1953 list_del(entry: &e->base.link);
1954 drm_vblank_put(dev, pipe);
1955 send_vblank_event(dev, e, seq, now);
1956 }
1957
1958 if (crtc && crtc->funcs->get_vblank_timestamp)
1959 high_prec = true;
1960
1961 trace_drm_vblank_event(crtc: pipe, seq, time: now, high_prec);
1962}
1963
1964/**
1965 * drm_handle_vblank - handle a vblank event
1966 * @dev: DRM device
1967 * @pipe: index of CRTC where this event occurred
1968 *
1969 * Drivers should call this routine in their vblank interrupt handlers to
1970 * update the vblank counter and send any signals that may be pending.
1971 *
1972 * This is the legacy version of drm_crtc_handle_vblank().
1973 */
1974bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1975{
1976 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1977 unsigned long irqflags;
1978 bool disable_irq;
1979
1980 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1981 return false;
1982
1983 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1984 return false;
1985
1986 spin_lock_irqsave(&dev->event_lock, irqflags);
1987
1988 /* Need timestamp lock to prevent concurrent execution with
1989 * vblank enable/disable, as this would cause inconsistent
1990 * or corrupted timestamps and vblank counts.
1991 */
1992 spin_lock(lock: &dev->vblank_time_lock);
1993
1994 /* Vblank irq handling disabled. Nothing to do. */
1995 if (!vblank->enabled) {
1996 spin_unlock(lock: &dev->vblank_time_lock);
1997 spin_unlock_irqrestore(lock: &dev->event_lock, flags: irqflags);
1998 return false;
1999 }
2000
2001 drm_update_vblank_count(dev, pipe, in_vblank_irq: true);
2002
2003 spin_unlock(lock: &dev->vblank_time_lock);
2004
2005 wake_up(&vblank->queue);
2006
2007 /* With instant-off, we defer disabling the interrupt until after
2008 * we finish processing the following vblank after all events have
2009 * been signaled. The disable has to be last (after
2010 * drm_handle_vblank_events) so that the timestamp is always accurate.
2011 */
2012 disable_irq = (dev->vblank_disable_immediate &&
2013 drm_vblank_offdelay > 0 &&
2014 !atomic_read(v: &vblank->refcount));
2015
2016 drm_handle_vblank_events(dev, pipe);
2017 drm_handle_vblank_works(vblank);
2018
2019 spin_unlock_irqrestore(lock: &dev->event_lock, flags: irqflags);
2020
2021 if (disable_irq)
2022 vblank_disable_fn(t: &vblank->disable_timer);
2023
2024 return true;
2025}
2026EXPORT_SYMBOL(drm_handle_vblank);
2027
2028/**
2029 * drm_crtc_handle_vblank - handle a vblank event
2030 * @crtc: where this event occurred
2031 *
2032 * Drivers should call this routine in their vblank interrupt handlers to
2033 * update the vblank counter and send any signals that may be pending.
2034 *
2035 * This is the native KMS version of drm_handle_vblank().
2036 *
2037 * Note that for a given vblank counter value drm_crtc_handle_vblank()
2038 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
2039 * provide a barrier: Any writes done before calling
2040 * drm_crtc_handle_vblank() will be visible to callers of the later
2041 * functions, if the vblank count is the same or a later one.
2042 *
2043 * See also &drm_vblank_crtc.count.
2044 *
2045 * Returns:
2046 * True if the event was successfully handled, false on failure.
2047 */
2048bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2049{
2050 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2051}
2052EXPORT_SYMBOL(drm_crtc_handle_vblank);
2053
2054/*
2055 * Get crtc VBLANK count.
2056 *
2057 * \param dev DRM device
2058 * \param data user argument, pointing to a drm_crtc_get_sequence structure.
2059 * \param file_priv drm file private for the user's open file descriptor
2060 */
2061
2062int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2063 struct drm_file *file_priv)
2064{
2065 struct drm_crtc *crtc;
2066 struct drm_vblank_crtc *vblank;
2067 int pipe;
2068 struct drm_crtc_get_sequence *get_seq = data;
2069 ktime_t now;
2070 bool vblank_enabled;
2071 int ret;
2072
2073 if (!drm_core_check_feature(dev, feature: DRIVER_MODESET))
2074 return -EOPNOTSUPP;
2075
2076 if (!drm_dev_has_vblank(dev))
2077 return -EOPNOTSUPP;
2078
2079 crtc = drm_crtc_find(dev, file_priv, id: get_seq->crtc_id);
2080 if (!crtc)
2081 return -ENOENT;
2082
2083 pipe = drm_crtc_index(crtc);
2084
2085 vblank = &dev->vblank[pipe];
2086 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
2087
2088 if (!vblank_enabled) {
2089 ret = drm_crtc_vblank_get(crtc);
2090 if (ret) {
2091 drm_dbg_core(dev,
2092 "crtc %d failed to acquire vblank counter, %d\n",
2093 pipe, ret);
2094 return ret;
2095 }
2096 }
2097 drm_modeset_lock(lock: &crtc->mutex, NULL);
2098 if (crtc->state)
2099 get_seq->active = crtc->state->enable;
2100 else
2101 get_seq->active = crtc->enabled;
2102 drm_modeset_unlock(lock: &crtc->mutex);
2103 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
2104 get_seq->sequence_ns = ktime_to_ns(kt: now);
2105 if (!vblank_enabled)
2106 drm_crtc_vblank_put(crtc);
2107 return 0;
2108}
2109
2110/*
2111 * Queue a event for VBLANK sequence
2112 *
2113 * \param dev DRM device
2114 * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2115 * \param file_priv drm file private for the user's open file descriptor
2116 */
2117
2118int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2119 struct drm_file *file_priv)
2120{
2121 struct drm_crtc *crtc;
2122 struct drm_vblank_crtc *vblank;
2123 int pipe;
2124 struct drm_crtc_queue_sequence *queue_seq = data;
2125 ktime_t now;
2126 struct drm_pending_vblank_event *e;
2127 u32 flags;
2128 u64 seq;
2129 u64 req_seq;
2130 int ret;
2131
2132 if (!drm_core_check_feature(dev, feature: DRIVER_MODESET))
2133 return -EOPNOTSUPP;
2134
2135 if (!drm_dev_has_vblank(dev))
2136 return -EOPNOTSUPP;
2137
2138 crtc = drm_crtc_find(dev, file_priv, id: queue_seq->crtc_id);
2139 if (!crtc)
2140 return -ENOENT;
2141
2142 flags = queue_seq->flags;
2143 /* Check valid flag bits */
2144 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2145 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2146 return -EINVAL;
2147
2148 pipe = drm_crtc_index(crtc);
2149
2150 vblank = &dev->vblank[pipe];
2151
2152 e = kzalloc(size: sizeof(*e), GFP_KERNEL);
2153 if (e == NULL)
2154 return -ENOMEM;
2155
2156 ret = drm_crtc_vblank_get(crtc);
2157 if (ret) {
2158 drm_dbg_core(dev,
2159 "crtc %d failed to acquire vblank counter, %d\n",
2160 pipe, ret);
2161 goto err_free;
2162 }
2163
2164 seq = drm_vblank_count_and_time(dev, pipe, vblanktime: &now);
2165 req_seq = queue_seq->sequence;
2166
2167 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2168 req_seq += seq;
2169
2170 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, ref: req_seq))
2171 req_seq = seq + 1;
2172
2173 e->pipe = pipe;
2174 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2175 e->event.base.length = sizeof(e->event.seq);
2176 e->event.seq.user_data = queue_seq->user_data;
2177
2178 spin_lock_irq(lock: &dev->event_lock);
2179
2180 /*
2181 * drm_crtc_vblank_off() might have been called after we called
2182 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2183 * vblank disable, so no need for further locking. The reference from
2184 * drm_crtc_vblank_get() protects against vblank disable from another source.
2185 */
2186 if (!READ_ONCE(vblank->enabled)) {
2187 ret = -EINVAL;
2188 goto err_unlock;
2189 }
2190
2191 ret = drm_event_reserve_init_locked(dev, file_priv, p: &e->base,
2192 e: &e->event.base);
2193
2194 if (ret)
2195 goto err_unlock;
2196
2197 e->sequence = req_seq;
2198
2199 if (drm_vblank_passed(seq, ref: req_seq)) {
2200 drm_crtc_vblank_put(crtc);
2201 send_vblank_event(dev, e, seq, now);
2202 queue_seq->sequence = seq;
2203 } else {
2204 /* drm_handle_vblank_events will call drm_vblank_put */
2205 list_add_tail(new: &e->base.link, head: &dev->vblank_event_list);
2206 queue_seq->sequence = req_seq;
2207 }
2208
2209 spin_unlock_irq(lock: &dev->event_lock);
2210 return 0;
2211
2212err_unlock:
2213 spin_unlock_irq(lock: &dev->event_lock);
2214 drm_crtc_vblank_put(crtc);
2215err_free:
2216 kfree(objp: e);
2217 return ret;
2218}
2219
2220

source code of linux/drivers/gpu/drm/drm_vblank.c