1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qsgvideonode_p.h"
5#include <QtQuick/qsgmaterial.h>
6#include "qsgvideotexture_p.h"
7#include <QtMultimedia/private/qvideotexturehelper_p.h>
8#include <private/qquicktextnode_p.h>
9#include <private/qquickvideooutput_p.h>
10#include <private/qabstractvideobuffer_p.h>
11#include <qmutex.h>
12
13QT_BEGIN_NAMESPACE
14
15/* Helpers */
16static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
17{
18 v->x = p.x();
19 v->y = p.y();
20}
21
22static inline void qSetTex(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
23{
24 v->tx = p.x();
25 v->ty = p.y();
26}
27
28static inline void qSwapTex(QSGGeometry::TexturedPoint2D *v0, QSGGeometry::TexturedPoint2D *v1)
29{
30 auto tvx = v0->tx;
31 auto tvy = v0->ty;
32 v0->tx = v1->tx;
33 v0->ty = v1->ty;
34 v1->tx = tvx;
35 v1->ty = tvy;
36}
37
38class QSGVideoMaterial;
39
40class QSGVideoMaterialRhiShader : public QSGMaterialShader
41{
42public:
43 QSGVideoMaterialRhiShader(const QVideoFrameFormat &format)
44 : m_format(format)
45 {
46 setShaderFileName(stage: VertexStage, filename: m_format.vertexShaderFileName());
47 setShaderFileName(stage: FragmentStage, filename: m_format.fragmentShaderFileName());
48 }
49
50 bool updateUniformData(RenderState &state, QSGMaterial *newMaterial,
51 QSGMaterial *oldMaterial) override;
52
53 void updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
54 QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
55
56protected:
57 QVideoFrameFormat m_format;
58 float m_planeWidth[3] = {0, 0, 0};
59 QMatrix4x4 m_colorMatrix;
60};
61
62class QSGVideoMaterial : public QSGMaterial
63{
64public:
65 QSGVideoMaterial(const QVideoFrameFormat &format);
66
67 [[nodiscard]] QSGMaterialType *type() const override {
68 static QSGMaterialType type[QVideoFrameFormat::NPixelFormats];
69 return &type[m_format.pixelFormat()];
70 }
71
72 [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override {
73 return new QSGVideoMaterialRhiShader(m_format);
74 }
75
76 int compare(const QSGMaterial *other) const override {
77 const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other);
78
79 qint64 diff = m_textures[0].comparisonKey() - m->m_textures[0].comparisonKey();
80 if (!diff)
81 diff = m_textures[1].comparisonKey() - m->m_textures[1].comparisonKey();
82 if (!diff)
83 diff = m_textures[2].comparisonKey() - m->m_textures[2].comparisonKey();
84
85 return diff < 0 ? -1 : (diff > 0 ? 1 : 0);
86 }
87
88 void updateBlending() {
89 // ### respect video formats with Alpha
90 setFlag(flags: Blending, on: !qFuzzyCompare(p1: m_opacity, p2: float(1.0)));
91 }
92
93 void setCurrentFrame(const QVideoFrame &frame) {
94 QMutexLocker lock(&m_frameMutex);
95 m_currentFrame = frame;
96 m_texturesDirty = true;
97 }
98
99 void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates);
100
101 QVideoFrameFormat m_format;
102 float m_planeWidth[3];
103 float m_opacity;
104
105 QMutex m_frameMutex;
106 bool m_texturesDirty = false;
107 QVideoFrame m_currentFrame;
108
109 enum { NVideoFrameSlots = 4 };
110 QVideoFrame m_videoFrameSlots[NVideoFrameSlots];
111 std::array<QSGVideoTexture, 3> m_textures;
112 std::unique_ptr<QVideoFrameTextures> m_videoFrameTextures;
113};
114
115void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates)
116{
117 QMutexLocker locker(&m_frameMutex);
118 if (!m_texturesDirty)
119 return;
120
121 // keep the video frames alive until we know that they are not needed anymore
122 Q_ASSERT(NVideoFrameSlots >= rhi->resourceLimit(QRhi::FramesInFlight));
123 m_videoFrameSlots[rhi->currentFrameSlot()] = m_currentFrame;
124
125 // update and upload all textures
126 m_videoFrameTextures = QVideoTextureHelper::createTextures(frame&: m_currentFrame, rhi, rub: resourceUpdates, oldTextures: std::move(m_videoFrameTextures));
127 if (!m_videoFrameTextures)
128 return;
129
130 for (int plane = 0; plane < 3; ++plane)
131 m_textures[plane].setRhiTexture(m_videoFrameTextures->texture(plane));
132 m_texturesDirty = false;
133}
134
135
136bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial,
137 QSGMaterial *oldMaterial)
138{
139 Q_UNUSED(oldMaterial);
140
141 auto m = static_cast<QSGVideoMaterial *>(newMaterial);
142
143 if (!state.isMatrixDirty() && !state.isOpacityDirty())
144 return false;
145
146 if (state.isOpacityDirty()) {
147 m->m_opacity = state.opacity();
148 m->updateBlending();
149 }
150
151 // Do this here, not in updateSampledImage. First, with multiple textures we want to
152 // do this once. More importantly, on some platforms (Android) the externalMatrix is
153 // updated by this function and we need that already in updateUniformData.
154 m->updateTextures(rhi: state.rhi(), resourceUpdates: state.resourceUpdateBatch());
155
156 m_format.updateUniformData(dst: state.uniformData(), frame: m->m_currentFrame,
157 transform: state.combinedMatrix(), opacity: state.opacity());
158
159 return true;
160}
161
162void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
163 QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
164{
165 Q_UNUSED(state);
166 Q_UNUSED(oldMaterial);
167 if (binding < 1 || binding > 3)
168 return;
169
170 auto m = static_cast<QSGVideoMaterial *>(newMaterial);
171 *texture = &m->m_textures[binding - 1];
172}
173
174QSGVideoMaterial::QSGVideoMaterial(const QVideoFrameFormat &format) :
175 m_format(format),
176 m_opacity(1.0)
177{
178 setFlag(flags: Blending, on: false);
179}
180
181QSGVideoNode::QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &format)
182 : m_parent(parent),
183 m_orientation(-1),
184 m_frameOrientation(-1),
185 m_frameMirrored(false),
186 m_format(format)
187{
188 setFlag(QSGNode::OwnsMaterial);
189 setFlag(QSGNode::OwnsGeometry);
190 m_material = new QSGVideoMaterial(format);
191 setMaterial(m_material);
192}
193
194QSGVideoNode::~QSGVideoNode()
195{
196 delete m_subtitleTextNode;
197}
198
199void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame)
200{
201 m_material->setCurrentFrame(frame);
202 markDirty(bits: DirtyMaterial);
203 updateSubtitle(frame);
204}
205
206void QSGVideoNode::updateSubtitle(const QVideoFrame &frame)
207{
208 QSize subtitleFrameSize = m_rect.size().toSize();
209 if (subtitleFrameSize.isEmpty())
210 return;
211 if (m_orientation % 180)
212 subtitleFrameSize.transpose();
213 if (!m_subtitleLayout.update(frameSize: subtitleFrameSize, text: frame.subtitleText()))
214 return;
215
216 delete m_subtitleTextNode;
217 m_subtitleTextNode = nullptr;
218 if (frame.subtitleText().isEmpty())
219 return;
220
221 m_subtitleTextNode = new QQuickTextNode(m_parent);
222 QColor bgColor = Qt::black;
223 bgColor.setAlpha(128);
224 m_subtitleTextNode->addRectangleNode(rect: m_subtitleLayout.bounds, color: bgColor);
225 m_subtitleTextNode->addTextLayout(position: m_subtitleLayout.layout.position(), textLayout: &m_subtitleLayout.layout, color: Qt::white);
226 appendChildNode(node: m_subtitleTextNode);
227 setSubtitleGeometry();
228}
229
230void QSGVideoNode::setSubtitleGeometry()
231{
232 if (!m_subtitleTextNode)
233 return;
234
235 if (m_material)
236 updateSubtitle(frame: m_material->m_currentFrame);
237
238 float rotate = -1.f * m_orientation;
239 float yTranslate = 0;
240 float xTranslate = 0;
241 if (m_orientation == 90) {
242 yTranslate = m_rect.height();
243 } else if (m_orientation == 180) {
244 yTranslate = m_rect.height();
245 xTranslate = m_rect.width();
246 } else if (m_orientation == 270) {
247 xTranslate = m_rect.width();
248 }
249
250 QMatrix4x4 transform;
251 transform.translate(x: m_rect.x() + xTranslate, y: m_rect.y() + yTranslate);
252 transform.rotate(angle: rotate, x: 0, y: 0, z: 1);
253
254 m_subtitleTextNode->setMatrix(transform);
255 m_subtitleTextNode->markDirty(bits: DirtyGeometry);
256}
257
258/* Update the vertices and texture coordinates. Orientation must be in {0,90,180,270} */
259void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &textureRect, int orientation)
260{
261 bool frameChanged = false;
262 if (m_material) {
263 if (m_material->m_currentFrame.rotationAngle() != m_frameOrientation
264 || m_material->m_currentFrame.mirrored() != m_frameMirrored) {
265 frameChanged = true;
266 }
267 }
268 if (rect == m_rect && textureRect == m_textureRect && orientation == m_orientation
269 && !frameChanged)
270 return;
271
272 m_rect = rect;
273 m_textureRect = textureRect;
274 m_orientation = orientation;
275 if (m_material) {
276 m_frameOrientation = m_material->m_currentFrame.rotationAngle();
277 m_frameMirrored = m_material->m_currentFrame.mirrored();
278 }
279 int videoRotation = orientation;
280 videoRotation += m_material ? m_material->m_currentFrame.rotationAngle() : 0;
281 videoRotation %= 360;
282
283 QSGGeometry *g = geometry();
284
285 if (g == nullptr)
286 g = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
287
288 QSGGeometry::TexturedPoint2D *v = g->vertexDataAsTexturedPoint2D();
289
290 // Set geometry first
291 qSetGeom(v: v + 0, p: rect.topLeft());
292 qSetGeom(v: v + 1, p: rect.bottomLeft());
293 qSetGeom(v: v + 2, p: rect.topRight());
294 qSetGeom(v: v + 3, p: rect.bottomRight());
295
296 // and then texture coordinates
297 switch (videoRotation) {
298 default:
299 // tl, bl, tr, br
300 qSetTex(v: v + 0, p: textureRect.topLeft());
301 qSetTex(v: v + 1, p: textureRect.bottomLeft());
302 qSetTex(v: v + 2, p: textureRect.topRight());
303 qSetTex(v: v + 3, p: textureRect.bottomRight());
304 break;
305
306 case 90:
307 // bl, br, tl, tr
308 qSetTex(v: v + 0, p: textureRect.bottomLeft());
309 qSetTex(v: v + 1, p: textureRect.bottomRight());
310 qSetTex(v: v + 2, p: textureRect.topLeft());
311 qSetTex(v: v + 3, p: textureRect.topRight());
312 break;
313
314 case 180:
315 // br, tr, bl, tl
316 qSetTex(v: v + 0, p: textureRect.bottomRight());
317 qSetTex(v: v + 1, p: textureRect.topRight());
318 qSetTex(v: v + 2, p: textureRect.bottomLeft());
319 qSetTex(v: v + 3, p: textureRect.topLeft());
320 break;
321
322 case 270:
323 // tr, tl, br, bl
324 qSetTex(v: v + 0, p: textureRect.topRight());
325 qSetTex(v: v + 1, p: textureRect.topLeft());
326 qSetTex(v: v + 2, p: textureRect.bottomRight());
327 qSetTex(v: v + 3, p: textureRect.bottomLeft());
328 break;
329 }
330
331 if (m_material && m_material->m_currentFrame.mirrored()) {
332 qSwapTex(v0: v + 0, v1: v + 2);
333 qSwapTex(v0: v + 1, v1: v + 3);
334 }
335
336 if (!geometry())
337 setGeometry(g);
338
339 markDirty(bits: DirtyGeometry);
340
341 setSubtitleGeometry();
342}
343
344QT_END_NAMESPACE
345

source code of qtmultimedia/src/multimediaquick/qsgvideonode_p.cpp