1// Copyright (C) 2021 The Qt Company Ltd.
2// Copyright (C) 2016 Research In Motion
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
4#include "qquickvideooutput_p.h"
5
6#include <private/qvideooutputorientationhandler_p.h>
7#include <private/qvideoframetexturepool_p.h>
8#include <QtMultimedia/qmediaplayer.h>
9#include <QtMultimedia/qmediacapturesession.h>
10#include <private/qfactoryloader_p.h>
11#include <QtCore/qloggingcategory.h>
12#include <QtQuick/QQuickWindow>
13#include <private/qquickwindow_p.h>
14#include <private/qmultimediautils_p.h>
15#include <qsgvideonode_p.h>
16#include <QtCore/qrunnable.h>
17
18QT_BEGIN_NAMESPACE
19
20Q_STATIC_LOGGING_CATEGORY(qLcVideo, "qt.multimedia.video")
21
22namespace {
23
24inline bool qIsDefaultAspect(int o)
25{
26 return (o % 180) == 0;
27}
28
29inline bool qIsDefaultAspect(QtVideo::Rotation rotation)
30{
31 return qIsDefaultAspect(o: qToUnderlying(e: rotation));
32}
33}
34
35/*!
36 \qmltype VideoOutput
37 //! \nativetype QQuickVideoOutput
38 \inherits Item
39 \brief Render video or camera viewfinder.
40
41 \ingroup multimedia_qml
42 \ingroup multimedia_video_qml
43 \inqmlmodule QtMultimedia
44
45 \qml
46
47 Rectangle {
48 width: 800
49 height: 600
50 color: "black"
51
52 MediaPlayer {
53 id: player
54 source: "file://video.webm"
55 videoOutput: videoOutput
56 }
57
58 VideoOutput {
59 id: videoOutput
60 anchors.fill: parent
61 }
62 }
63
64 \endqml
65
66 The VideoOutput item supports untransformed, stretched, and uniformly scaled video presentation.
67 For a description of stretched uniformly scaled presentation, see the \l fillMode property
68 description.
69
70 \sa MediaPlayer, Camera
71
72\omit
73 \section1 Screen Saver
74
75 If it is likely that an application will be playing video for an extended
76 period of time without user interaction it may be necessary to disable
77 the platform's screen saver. The \l ScreenSaver (from \l QtSystemInfo)
78 may be used to disable the screensaver in this fashion:
79
80 \qml
81 import QtSystemInfo
82
83 ScreenSaver { screenSaverEnabled: false }
84 \endqml
85\endomit
86*/
87
88// TODO: Restore Qt System Info docs when the module is released
89
90/*!
91 \internal
92 \class QQuickVideoOutput
93 \brief The QQuickVideoOutput class provides a video output item.
94*/
95
96QQuickVideoOutput::QQuickVideoOutput(QQuickItem *parent) :
97 QQuickItem(parent)
98{
99 setFlag(flag: ItemHasContents, enabled: true);
100
101 m_sink = new QQuickVideoSink(this);
102 qRegisterMetaType<QVideoFrameFormat>();
103
104 // TODO: investigate if we have any benefit of setting frame in the source thread
105 connect(sender: m_sink, signal: &QVideoSink::videoFrameChanged, context: this,
106 slot: makeGuardedCall(f: [this](const QVideoFrame &frame) {
107 if (frame.isValid() || m_endOfStreamPolicy == ClearOutput)
108 setFrame(frame);
109 }),
110 type: Qt::DirectConnection);
111
112 initRhiForSink();
113}
114
115QQuickVideoOutput::~QQuickVideoOutput()
116{
117 {
118 QMutexLocker lock(&m_destructorGuard->m_mutex);
119 m_destructorGuard->m_isAlive = false;
120 }
121
122 delete m_sink;
123 disconnectWindowConnections();
124}
125
126/*!
127 \qmlproperty object QtMultimedia::VideoOutput::videoSink
128
129 This property holds the underlaying C++ QVideoSink object that is used
130 to render the video frames to this VideoOutput element.
131
132 Normal usage of VideoOutput from QML should not require using this property.
133*/
134
135QVideoSink *QQuickVideoOutput::videoSink() const
136{
137 return m_sink;
138}
139
140/*!
141 \qmlproperty enumeration QtMultimedia::VideoOutput::fillMode
142
143 Set this property to define how the video is scaled to fit the target area.
144
145 \list
146 \li Stretch - the video is scaled to fit.
147 \li PreserveAspectFit - the video is scaled uniformly to fit without cropping
148 \li PreserveAspectCrop - the video is scaled uniformly to fill, cropping if necessary
149 \endlist
150
151 The default fill mode is PreserveAspectFit.
152*/
153
154QQuickVideoOutput::FillMode QQuickVideoOutput::fillMode() const
155{
156 return FillMode(m_aspectRatioMode);
157}
158
159void QQuickVideoOutput::setFillMode(FillMode mode)
160{
161 if (mode == fillMode())
162 return;
163
164 m_aspectRatioMode = Qt::AspectRatioMode(mode);
165
166 m_geometryDirty = true;
167 update();
168
169 emit fillModeChanged(mode);
170}
171
172void QQuickVideoOutput::_q_newFrame(QSize size)
173{
174 update();
175
176 size = qRotatedFrameSize(size, rotation: m_frameDisplayingRotation);
177
178 if (m_nativeSize != size) {
179 m_nativeSize = size;
180
181 m_geometryDirty = true;
182
183 setImplicitWidth(size.width());
184 setImplicitHeight(size.height());
185
186 emit sourceRectChanged();
187 }
188}
189
190/* Based on fill mode and our size, figure out the source/dest rects */
191void QQuickVideoOutput::_q_updateGeometry()
192{
193 const QRectF rect(0, 0, width(), height());
194 const QRectF absoluteRect(x(), y(), width(), height());
195
196 if (!m_geometryDirty && m_lastRect == absoluteRect)
197 return;
198
199 QRectF oldContentRect(m_contentRect);
200
201 m_geometryDirty = false;
202 m_lastRect = absoluteRect;
203
204 const auto fill = m_aspectRatioMode;
205 if (m_nativeSize.isEmpty()) {
206 //this is necessary for item to receive the
207 //first paint event and configure video surface.
208 m_contentRect = rect;
209 } else if (fill == Qt::IgnoreAspectRatio) {
210 m_contentRect = rect;
211 } else {
212 QSizeF scaled = m_nativeSize;
213 scaled.scale(s: rect.size(), mode: fill);
214
215 m_contentRect = QRectF(QPointF(), scaled);
216 m_contentRect.moveCenter(p: rect.center());
217 }
218
219 updateGeometry();
220
221 if (m_contentRect != oldContentRect)
222 emit contentRectChanged();
223}
224
225/*!
226 \qmlproperty int QtMultimedia::VideoOutput::orientation
227
228 This property determines the angle in, degrees, at which the displayed video
229 is rotated clockwise in video coordinates, where the Y-axis points
230 downwards on the display.
231 The orientation transformation is applied before \l mirrored.
232
233 The orientation change affects the mapping of coordinates from the source to the viewport.
234
235 Only multiples of \c 90 degrees are supported, that is 0, 90, -90, 180, 270, etc.,
236 otherwise, the specified value is ignored.
237
238 In some cases, the source video stream requires a certain
239 orientation to be corrected. This includes
240 sources like a camera viewfinder, where the displayed
241 viewfinder should match the reality, no matter what rotation
242 the rest of the user interface has.
243
244 We recommend using this property to compensate a user interface
245 rotation, or align the output view with other application business
246 requirements.
247
248 The default value is \c 0.
249*/
250int QQuickVideoOutput::orientation() const
251{
252 return m_orientation;
253}
254
255void QQuickVideoOutput::setOrientation(int orientation)
256{
257 // Make sure it's a multiple of 90.
258 if (orientation % 90)
259 return;
260
261 // If there's no actual change, return
262 if (m_orientation == orientation)
263 return;
264
265 // If the new orientation is the same effect
266 // as the old one, don't update the video node stuff
267 if (qVideoRotationFromDegrees(clockwiseDegrees: orientation - m_orientation) == QtVideo::Rotation::None) {
268 m_orientation = orientation;
269 emit orientationChanged();
270 return;
271 }
272
273 m_geometryDirty = true;
274
275 // Otherwise, a new orientation
276 // See if we need to change aspect ratio orientation too
277 bool oldAspect = qIsDefaultAspect(o: m_orientation);
278 bool newAspect = qIsDefaultAspect(o: orientation);
279
280 m_orientation = orientation;
281
282 {
283 QMutexLocker lock(&m_frameMutex);
284 m_frameDisplayingRotation = qNormalizedFrameTransformation(frame: m_frame, videoOutputRotation: m_orientation).rotation;
285 }
286
287 if (oldAspect != newAspect) {
288 m_nativeSize.transpose();
289
290 setImplicitWidth(m_nativeSize.width());
291 setImplicitHeight(m_nativeSize.height());
292
293 // Source rectangle does not change for orientation
294 }
295
296 update();
297 emit orientationChanged();
298}
299
300/*!
301 \qmlproperty bool QtMultimedia::VideoOutput::mirrored
302 \since 6.9
303
304 Determines whether the displayed video is mirrored around its vertical axis.
305 We recommend using this property if the transformation of the source video stream,
306 such as from a front camera, does not align with the application's business logic.
307 The mirroring is applied after \l orientation.
308
309 The property change affects the mapping of coordinates from the source to the viewport.
310
311 The default value is \c false.
312*/
313bool QQuickVideoOutput::mirrored() const
314{
315 return m_mirrored;
316}
317
318void QQuickVideoOutput::setMirrored(bool mirrored)
319{
320 if (m_mirrored == mirrored)
321 return;
322 m_mirrored = mirrored;
323
324 update();
325 emit mirroredChanged();
326}
327
328/*!
329 \qmlproperty rectangle QtMultimedia::VideoOutput::contentRect
330
331 This property holds the item coordinates of the area that
332 would contain video to render. With certain fill modes,
333 this rectangle will be larger than the visible area of the
334 \c VideoOutput.
335
336 This property is useful when other coordinates are specified
337 in terms of the source dimensions - this applied for relative
338 (normalized) frame coordinates in the range of 0 to 1.0.
339
340 Areas outside this will be transparent.
341*/
342QRectF QQuickVideoOutput::contentRect() const
343{
344 return m_contentRect;
345}
346
347/*!
348 \qmlproperty enumeration QtMultimedia::VideoOutput::endOfStreamPolicy
349 \since 6.9
350
351 This property specifies the policy to apply when the video stream ends. This
352 occurs at the end of media playback or upon deactivation of the camera, screen,
353 or window capture.
354
355 The \c endOfStreamPolicy can be one of:
356
357 \value ClearOutput The video output is cleared.
358 \value KeepLastFrame The video output continues displaying the
359 last frame. Use the method \l clearOutput() to clear the output manually.
360
361 The default value is \c ClearOutput.
362*/
363QQuickVideoOutput::EndOfStreamPolicy QQuickVideoOutput::endOfStreamPolicy() const
364{
365 return m_endOfStreamPolicy;
366}
367
368void QQuickVideoOutput::setEndOfStreamPolicy(EndOfStreamPolicy policy)
369{
370 if (m_endOfStreamPolicy == policy)
371 return;
372
373 m_endOfStreamPolicy = policy;
374 emit endOfStreamPolicyChanged(policy);
375}
376
377/*!
378 \qmlmethod void QtMultimedia::VideoOutput::clearOutput
379 \since 6.9
380
381 Clears the video output by removing the currently displayed video frame.
382 This method is recommended when you need to remove the last video frame after
383 detaching the video output from the source or after the source stream ends
384 with the video output's \l endOfStreamPolicy property set to \c KeepLastFrame.
385*/
386void QQuickVideoOutput::clearOutput()
387{
388 setFrame({});
389}
390
391/*!
392 \qmlproperty rectangle QtMultimedia::VideoOutput::sourceRect
393
394 This property holds the area of the source video
395 content that is considered for rendering. The
396 values are in source pixel coordinates, adjusted for
397 the source's pixel aspect ratio.
398
399 Note that typically the top left corner of this rectangle
400 will be \c {0,0} while the width and height will be the
401 width and height of the input content. Only when the video
402 source has a viewport set, these values will differ.
403
404 The orientation setting does not affect this rectangle.
405
406 \sa QVideoFrameFormat::viewport()
407*/
408QRectF QQuickVideoOutput::sourceRect() const
409{
410 // We might have to transpose back
411 QSizeF size = m_nativeSize;
412 if (!size.isValid())
413 return {};
414
415 if (!qIsDefaultAspect(rotation: m_frameDisplayingRotation))
416 size.transpose();
417
418
419 // Take the viewport into account for the top left position.
420 // m_nativeSize is already adjusted to the viewport, as it originates
421 // from QVideoFrameFormat::viewport(), which includes pixel aspect ratio
422 const QRectF viewport = adjustedViewport();
423 Q_ASSERT(viewport.size() == size);
424 return QRectF(viewport.topLeft(), size);
425}
426
427void QQuickVideoOutput::geometryChange(const QRectF &newGeometry, const QRectF &oldGeometry)
428{
429 Q_UNUSED(newGeometry);
430 Q_UNUSED(oldGeometry);
431
432 QQuickItem::geometryChange(newGeometry, oldGeometry);
433
434 // Explicitly listen to geometry changes here. This is needed since changing the position does
435 // not trigger a call to updatePaintNode().
436 // We need to react to position changes though, as the window backened's display rect gets
437 // changed in that situation.
438 _q_updateGeometry();
439}
440
441void QQuickVideoOutput::releaseResources()
442{
443 // Called on the gui thread when the window is closed or changed.
444 initRhiForSink();
445 QQuickItem::releaseResources();
446}
447
448void QQuickVideoOutput::initRhiForSink()
449{
450 QRhi *rhi = m_window ? QQuickWindowPrivate::get(c: m_window)->rhi : nullptr;
451 m_sink->setRhi(rhi);
452}
453
454void QQuickVideoOutput::itemChange(QQuickItem::ItemChange change,
455 const QQuickItem::ItemChangeData &changeData)
456{
457 if (change != QQuickItem::ItemSceneChange)
458 return;
459
460 if (changeData.window == m_window)
461 return;
462
463 disconnectWindowConnections();
464 m_window = changeData.window;
465
466 if (m_window) {
467 auto connectToWindow = [&](auto signal, auto function) {
468 connect(m_window, signal, this, makeGuardedCall(std::move(function)),
469 Qt::DirectConnection);
470 };
471
472 // We want to receive the signals in the render thread
473 connectToWindow(&QQuickWindow::sceneGraphInitialized, [this] {
474 initRhiForSink();
475 });
476
477 connectToWindow(&QQuickWindow::sceneGraphInvalidated, [this] {
478 if (auto texturePool = m_texturePool.lock())
479 texturePool->clearTextures();
480 m_sink->setRhi(nullptr);
481 });
482
483 connectToWindow(&QQuickWindow::afterFrameEnd, [this] {
484 if (auto texturePool = m_texturePool.lock())
485 texturePool->onFrameEndInvoked();
486 });
487 }
488 initRhiForSink();
489}
490
491QSize QQuickVideoOutput::nativeSize() const
492{
493 return m_videoFormat.viewport().size();
494}
495
496void QQuickVideoOutput::updateGeometry()
497{
498 const QRectF viewport = m_videoFormat.viewport();
499 const QSizeF frameSize = m_videoFormat.frameSize();
500 const QRectF normalizedViewport(viewport.x() / frameSize.width(),
501 viewport.y() / frameSize.height(),
502 viewport.width() / frameSize.width(),
503 viewport.height() / frameSize.height());
504 const QRectF rect(0, 0, width(), height());
505 if (nativeSize().isEmpty()) {
506 m_renderedRect = rect;
507 m_sourceTextureRect = normalizedViewport;
508 } else if (m_aspectRatioMode == Qt::IgnoreAspectRatio) {
509 m_renderedRect = rect;
510 m_sourceTextureRect = normalizedViewport;
511 } else if (m_aspectRatioMode == Qt::KeepAspectRatio) {
512 m_sourceTextureRect = normalizedViewport;
513 m_renderedRect = contentRect();
514 } else if (m_aspectRatioMode == Qt::KeepAspectRatioByExpanding) {
515 m_renderedRect = rect;
516 const qreal contentHeight = contentRect().height();
517 const qreal contentWidth = contentRect().width();
518
519 // Calculate the size of the source rectangle without taking the viewport into account
520 const qreal relativeOffsetLeft = -contentRect().left() / contentWidth;
521 const qreal relativeOffsetTop = -contentRect().top() / contentHeight;
522 const qreal relativeWidth = rect.width() / contentWidth;
523 const qreal relativeHeight = rect.height() / contentHeight;
524
525 // Now take the viewport size into account
526 const qreal totalOffsetLeft = normalizedViewport.x() + relativeOffsetLeft * normalizedViewport.width();
527 const qreal totalOffsetTop = normalizedViewport.y() + relativeOffsetTop * normalizedViewport.height();
528 const qreal totalWidth = normalizedViewport.width() * relativeWidth;
529 const qreal totalHeight = normalizedViewport.height() * relativeHeight;
530
531 if (qIsDefaultAspect(rotation: m_frameDisplayingRotation)) {
532 m_sourceTextureRect = QRectF(totalOffsetLeft, totalOffsetTop,
533 totalWidth, totalHeight);
534 } else {
535 m_sourceTextureRect = QRectF(totalOffsetTop, totalOffsetLeft,
536 totalHeight, totalWidth);
537 }
538 }
539}
540
541QSGNode *QQuickVideoOutput::updatePaintNode(QSGNode *oldNode,
542 QQuickItem::UpdatePaintNodeData *data)
543{
544 Q_UNUSED(data);
545 _q_updateGeometry();
546
547 QSGVideoNode *videoNode = static_cast<QSGVideoNode *>(oldNode);
548
549 QMutexLocker lock(&m_frameMutex);
550
551 if (m_frameChanged) {
552 if (videoNode && videoNode->pixelFormat() != m_frame.pixelFormat()) {
553 qCDebug(qLcVideo) << "updatePaintNode: deleting old video node because frame format changed";
554 delete videoNode;
555 videoNode = nullptr;
556 }
557
558 if (!m_frame.isValid()) {
559 qCDebug(qLcVideo) << "updatePaintNode: no frames yet";
560 m_frameChanged = false;
561 return nullptr;
562 }
563
564 if (!videoNode) {
565 // Get a node that supports our frame. The surface is irrelevant, our
566 // QSGVideoItemSurface supports (logically) anything.
567 updateGeometry();
568 QRhi *rhi = m_window ? QQuickWindowPrivate::get(c: m_window)->rhi : nullptr;
569 videoNode = new QSGVideoNode(this, m_videoFormat, rhi);
570 m_texturePool = videoNode->texturePool();
571 qCDebug(qLcVideo) << "updatePaintNode: Video node created. Handle type:" << m_frame.handleType();
572 }
573 }
574
575 if (!videoNode) {
576 m_frameChanged = false;
577 m_frame = QVideoFrame();
578 return nullptr;
579 }
580
581 if (m_frameChanged) {
582 videoNode->setCurrentFrame(m_frame);
583
584 updateHdr(videoNode);
585
586 //don't keep the frame for more than really necessary
587 m_frameChanged = false;
588 m_frame = QVideoFrame();
589 }
590
591 videoNode->setTexturedRectGeometry(
592 boundingRect: m_renderedRect, textureRect: m_sourceTextureRect,
593 videoOutputTransformation: VideoTransformation{ .rotation: qVideoRotationFromDegrees(clockwiseDegrees: orientation()), .mirroredHorizontallyAfterRotation: m_mirrored });
594
595 return videoNode;
596}
597
598void QQuickVideoOutput::updateHdr(QSGVideoNode *videoNode)
599{
600 auto *videoOutputWindow = window();
601 if (!videoOutputWindow)
602 return;
603
604 auto *swapChain = videoOutputWindow->swapChain();
605 if (!swapChain)
606 return;
607
608 const auto requiredSwapChainFormat = qGetRequiredSwapChainFormat(format: m_frame.surfaceFormat());
609 if (qShouldUpdateSwapChainFormat(swapChain, requiredSwapChainFormat)) {
610 auto *recreateSwapChainJob = QRunnable::create(functionToRun: [swapChain, requiredSwapChainFormat]() {
611 swapChain->destroy();
612 swapChain->setFormat(requiredSwapChainFormat);
613 swapChain->createOrResize();
614 });
615
616 // Even though the 'recreate swap chain' job is scheduled for the current frame the
617 // effect will be visible only starting from the next frame since the recreation would
618 // happen after the actual swap.
619 videoOutputWindow->scheduleRenderJob(job: recreateSwapChainJob, schedule: QQuickWindow::AfterSwapStage);
620 }
621
622 videoNode->setSurfaceFormat(swapChain->format());
623 videoNode->setHdrInfo(swapChain->hdrInfo());
624}
625
626void QQuickVideoOutput::disconnectWindowConnections()
627{
628 if (!m_window)
629 return;
630
631 m_window->disconnect(receiver: this);
632}
633
634QRectF QQuickVideoOutput::adjustedViewport() const
635{
636 return m_videoFormat.viewport();
637}
638
639void QQuickVideoOutput::setFrame(const QVideoFrame &frame)
640{
641 {
642 QMutexLocker lock(&m_frameMutex);
643
644 m_videoFormat = frame.surfaceFormat();
645 m_frame = frame;
646 m_frameDisplayingRotation = qNormalizedFrameTransformation(frame, videoOutputRotation: m_orientation).rotation;
647 m_frameChanged = true;
648 }
649
650 QMetaObject::invokeMethod(object: this, function: &QQuickVideoOutput::_q_newFrame, args: frame.size());
651}
652
653std::optional<std::chrono::nanoseconds> QQuickVideoOutput::g_signalBackoff;
654void QQuickVideoOutput::setSignalBackoff(std::optional<std::chrono::nanoseconds> ns)
655{
656 g_signalBackoff = ns;
657}
658
659QT_END_NAMESPACE
660
661#include "moc_qquickvideooutput_p.cpp"
662

source code of qtmultimedia/src/multimediaquick/qquickvideooutput.cpp