1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qsgvideonode_p.h"
5#include <QtQuick/qsgmaterial.h>
6#include "qsgvideotexture_p.h"
7#include <QtMultimedia/private/qvideotexturehelper_p.h>
8#include <private/qsginternaltextnode_p.h>
9#include <private/qquickitem_p.h>
10#include <private/qquickvideooutput_p.h>
11#include <private/qhwvideobuffer_p.h>
12#include <private/qvideoframetexturepool_p.h>
13
14QT_BEGIN_NAMESPACE
15
16/* Helpers */
17static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
18{
19 v->x = p.x();
20 v->y = p.y();
21}
22
23static inline void qSetTex(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
24{
25 v->tx = p.x();
26 v->ty = p.y();
27}
28
29static inline void qSwapTex(QSGGeometry::TexturedPoint2D *v0, QSGGeometry::TexturedPoint2D *v1)
30{
31 auto tvx = v0->tx;
32 auto tvy = v0->ty;
33 v0->tx = v1->tx;
34 v0->ty = v1->ty;
35 v1->tx = tvx;
36 v1->ty = tvy;
37}
38
39class QSGVideoMaterial;
40
41class QSGVideoMaterialRhiShader : public QSGMaterialShader
42{
43public:
44 QSGVideoMaterialRhiShader(const QVideoFrameFormat &videoFormat,
45 const QRhiSwapChain::Format surfaceFormat,
46 const QRhiSwapChainHdrInfo &hdrInfo,
47 QRhi *rhi)
48 : m_videoFormat(videoFormat)
49 , m_surfaceFormat(surfaceFormat)
50 , m_hdrInfo(hdrInfo)
51 {
52 setShaderFileName(stage: VertexStage, filename: QVideoTextureHelper::vertexShaderFileName(format: m_videoFormat));
53 setShaderFileName(stage: FragmentStage,
54 filename: QVideoTextureHelper::fragmentShaderFileName(
55 format: m_videoFormat, rhi, surfaceFormat: m_surfaceFormat));
56 }
57
58 bool updateUniformData(RenderState &state, QSGMaterial *newMaterial,
59 QSGMaterial *oldMaterial) override;
60
61 void updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
62 QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
63
64protected:
65 QVideoFrameFormat m_videoFormat;
66 QRhiSwapChain::Format m_surfaceFormat;
67 QRhiSwapChainHdrInfo m_hdrInfo;
68};
69
70class QSGVideoMaterial : public QSGMaterial
71{
72public:
73 QSGVideoMaterial(const QVideoFrameFormat &videoFormat, QRhi *rhi);
74
75 [[nodiscard]] QSGMaterialType *type() const override {
76 static constexpr int NFormats = QRhiSwapChain::HDRExtendedDisplayP3Linear + 1;
77 static QSGMaterialType type[QVideoFrameFormat::NPixelFormats][NFormats];
78 return &type[m_videoFormat.pixelFormat()][m_surfaceFormat];
79 }
80
81 [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override {
82 return new QSGVideoMaterialRhiShader(m_videoFormat, m_surfaceFormat, m_hdrInfo, m_rhi);
83 }
84
85 int compare(const QSGMaterial *other) const override {
86 const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other);
87
88 qint64 diff = m_textures[0].comparisonKey() - m->m_textures[0].comparisonKey();
89 if (!diff)
90 diff = m_textures[1].comparisonKey() - m->m_textures[1].comparisonKey();
91 if (!diff)
92 diff = m_textures[2].comparisonKey() - m->m_textures[2].comparisonKey();
93
94 return diff < 0 ? -1 : (diff > 0 ? 1 : 0);
95 }
96
97 void updateBlending() {
98 // ### respect video formats with Alpha
99 setFlag(flags: Blending, on: !qFuzzyCompare(p1: m_opacity, p2: float(1.0)));
100 }
101
102 void setSurfaceFormat(const QRhiSwapChain::Format surfaceFormat)
103 {
104 m_surfaceFormat = surfaceFormat;
105 }
106
107 void setHdrInfo(const QRhiSwapChainHdrInfo &hdrInfo)
108 {
109 m_hdrInfo = hdrInfo;
110 }
111
112 void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates);
113
114 QVideoFrameFormat m_videoFormat;
115 QRhiSwapChain::Format m_surfaceFormat = QRhiSwapChain::SDR;
116 float m_opacity = 1.0f;
117 QRhiSwapChainHdrInfo m_hdrInfo;
118
119 QVideoFrameTexturePoolPtr m_texturePool = std::make_shared<QVideoFrameTexturePool>();
120 std::array<QSGVideoTexture, 3> m_textures;
121
122 QRhi *m_rhi;
123};
124
125void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates)
126{
127 if (!m_texturePool->texturesDirty())
128 return;
129
130 QVideoFrameTextures *textures = m_texturePool->updateTextures(rhi&: *rhi, rub&: *resourceUpdates);
131 if (!textures)
132 return;
133
134 for (int plane = 0; plane < 3; ++plane)
135 m_textures[plane].setRhiTexture(textures->texture(plane));
136}
137
138
139bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial,
140 QSGMaterial *oldMaterial)
141{
142 Q_UNUSED(oldMaterial);
143
144 auto m = static_cast<QSGVideoMaterial *>(newMaterial);
145
146 if (!state.isMatrixDirty() && !state.isOpacityDirty())
147 return false;
148
149 if (state.isOpacityDirty()) {
150 m->m_opacity = state.opacity();
151 m->updateBlending();
152 }
153
154 // Do this here, not in updateSampledImage. First, with multiple textures we want to
155 // do this once. More importantly, on some platforms (Android) the externalMatrix is
156 // updated by this function and we need that already in updateUniformData.
157 m->updateTextures(rhi: state.rhi(), resourceUpdates: state.resourceUpdateBatch());
158
159 float maxNits = 100; // Default to de-facto SDR nits
160 if (m_surfaceFormat == QRhiSwapChain::HDRExtendedSrgbLinear) {
161 if (m_hdrInfo.limitsType == QRhiSwapChainHdrInfo::ColorComponentValue)
162 maxNits = 100 * m_hdrInfo.limits.colorComponentValue.maxColorComponentValue;
163 else
164 maxNits = m_hdrInfo.limits.luminanceInNits.maxLuminance;
165 }
166
167 QVideoTextureHelper::updateUniformData(dst: state.uniformData(), rhi: m->m_rhi, format: m_videoFormat,
168 frame: m->m_texturePool->currentFrame(), transform: state.combinedMatrix(),
169 opacity: state.opacity(), maxNits);
170
171 return true;
172}
173
174void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
175 QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
176{
177 Q_UNUSED(state);
178 Q_UNUSED(oldMaterial);
179 if (binding < 1 || binding > 3)
180 return;
181
182 auto m = static_cast<QSGVideoMaterial *>(newMaterial);
183 *texture = &m->m_textures[binding - 1];
184}
185
186QSGVideoMaterial::QSGVideoMaterial(const QVideoFrameFormat &videoFormat, QRhi *rhi)
187 : m_videoFormat(videoFormat),
188 m_rhi(rhi)
189{
190 setFlag(flags: Blending, on: false);
191}
192
193QSGVideoNode::QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &videoFormat,
194 QRhi *rhi)
195 : m_parent(parent), m_videoFormat(videoFormat)
196{
197 setFlag(QSGNode::OwnsMaterial);
198 setFlag(QSGNode::OwnsGeometry);
199 m_material = new QSGVideoMaterial(videoFormat, rhi);
200 setMaterial(m_material);
201}
202
203QSGVideoNode::~QSGVideoNode()
204{
205 delete m_subtitleTextNode;
206}
207
208void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame)
209{
210 texturePool()->setCurrentFrame(frame);
211 markDirty(bits: DirtyMaterial);
212 updateSubtitle(frame);
213}
214
215void QSGVideoNode::setSurfaceFormat(const QRhiSwapChain::Format surfaceFormat)
216{
217 m_material->setSurfaceFormat(surfaceFormat);
218 markDirty(bits: DirtyMaterial);
219}
220
221void QSGVideoNode::setHdrInfo(const QRhiSwapChainHdrInfo &hdrInfo)
222{
223 m_material->setHdrInfo(hdrInfo);
224 markDirty(bits: DirtyMaterial);
225}
226
227void QSGVideoNode::updateSubtitle(const QVideoFrame &frame)
228{
229 QSize subtitleFrameSize = m_rect.size().toSize();
230 if (subtitleFrameSize.isEmpty())
231 return;
232
233 subtitleFrameSize = qRotatedFrameSize(size: subtitleFrameSize, rotation: m_videoOutputTransformation.rotation);
234
235 if (!m_subtitleLayout.update(frameSize: subtitleFrameSize, text: frame.subtitleText()))
236 return;
237
238 delete m_subtitleTextNode;
239 m_subtitleTextNode = nullptr;
240 if (frame.subtitleText().isEmpty())
241 return;
242
243 QQuickItemPrivate *parent_d = QQuickItemPrivate::get(item: m_parent);
244
245 m_subtitleTextNode = parent_d->sceneGraphContext()->createInternalTextNode(renderContext: parent_d->sceneGraphRenderContext());
246 m_subtitleTextNode->setColor(Qt::white);
247 QColor bgColor = Qt::black;
248 bgColor.setAlpha(128);
249 m_subtitleTextNode->addRectangleNode(rect: m_subtitleLayout.bounds, color: bgColor);
250 m_subtitleTextNode->addTextLayout(position: m_subtitleLayout.layout.position(), layout: &m_subtitleLayout.layout);
251 appendChildNode(node: m_subtitleTextNode);
252 setSubtitleGeometry();
253}
254
255void QSGVideoNode::setSubtitleGeometry()
256{
257 if (!m_subtitleTextNode)
258 return;
259
260 if (m_material)
261 updateSubtitle(frame: texturePool()->currentFrame());
262
263 float rotate = -1.f * qToUnderlying(e: m_videoOutputTransformation.rotation);
264 float yTranslate = 0;
265 float xTranslate = 0;
266 if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise90) {
267 yTranslate = m_rect.height();
268 } else if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise180) {
269 yTranslate = m_rect.height();
270 xTranslate = m_rect.width();
271 } else if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise270) {
272 xTranslate = m_rect.width();
273 }
274
275 QMatrix4x4 transform;
276 transform.translate(x: m_rect.x() + xTranslate, y: m_rect.y() + yTranslate);
277 transform.rotate(angle: rotate, x: 0, y: 0, z: 1);
278 // TODO: Investigate if we should we mirror subtitles
279 // if (m_videoOutputTransformation.mirrorredHorizontallyAfterRotation)
280 // transform.scale(-1.f, 1.f);
281
282 m_subtitleTextNode->setMatrix(transform);
283 m_subtitleTextNode->markDirty(bits: DirtyGeometry);
284}
285
286/* Update the vertices and texture coordinates.*/
287void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &textureRect,
288 VideoTransformation videoOutputTransformation)
289{
290 const VideoTransformation currentFrameTransformation = qNormalizedFrameTransformation(
291 frame: m_material ? texturePool()->currentFrame() : QVideoFrame{}, videoOutputTransformation);
292
293 if (rect == m_rect && textureRect == m_textureRect
294 && videoOutputTransformation == m_videoOutputTransformation
295 && currentFrameTransformation == m_frameTransformation)
296 return;
297
298 m_rect = rect;
299 m_textureRect = textureRect;
300 m_videoOutputTransformation = videoOutputTransformation;
301 m_frameTransformation = currentFrameTransformation;
302
303 QSGGeometry *g = geometry();
304
305 if (g == nullptr)
306 g = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
307
308 QSGGeometry::TexturedPoint2D *v = g->vertexDataAsTexturedPoint2D();
309
310 // Vertexes:
311 // 0 2
312 //
313 // 1 3
314
315 // Set geometry first
316 qSetGeom(v: v + 0, p: rect.topLeft());
317 qSetGeom(v: v + 1, p: rect.bottomLeft());
318 qSetGeom(v: v + 2, p: rect.topRight());
319 qSetGeom(v: v + 3, p: rect.bottomRight());
320
321 // and then texture coordinates
322 switch (currentFrameTransformation.rotation) {
323 default:
324 // tl, bl, tr, br
325 qSetTex(v: v + 0, p: textureRect.topLeft());
326 qSetTex(v: v + 1, p: textureRect.bottomLeft());
327 qSetTex(v: v + 2, p: textureRect.topRight());
328 qSetTex(v: v + 3, p: textureRect.bottomRight());
329 break;
330
331 case QtVideo::Rotation::Clockwise90:
332 // bl, br, tl, tr
333 qSetTex(v: v + 0, p: textureRect.bottomLeft());
334 qSetTex(v: v + 1, p: textureRect.bottomRight());
335 qSetTex(v: v + 2, p: textureRect.topLeft());
336 qSetTex(v: v + 3, p: textureRect.topRight());
337 break;
338
339 case QtVideo::Rotation::Clockwise180:
340 // br, tr, bl, tl
341 qSetTex(v: v + 0, p: textureRect.bottomRight());
342 qSetTex(v: v + 1, p: textureRect.topRight());
343 qSetTex(v: v + 2, p: textureRect.bottomLeft());
344 qSetTex(v: v + 3, p: textureRect.topLeft());
345 break;
346
347 case QtVideo::Rotation::Clockwise270:
348 // tr, tl, br, bl
349 qSetTex(v: v + 0, p: textureRect.topRight());
350 qSetTex(v: v + 1, p: textureRect.topLeft());
351 qSetTex(v: v + 2, p: textureRect.bottomRight());
352 qSetTex(v: v + 3, p: textureRect.bottomLeft());
353 break;
354 }
355
356 if (m_frameTransformation.mirrorredHorizontallyAfterRotation) {
357 qSwapTex(v0: v + 0, v1: v + 2);
358 qSwapTex(v0: v + 1, v1: v + 3);
359 }
360
361 if (!geometry())
362 setGeometry(g);
363
364 markDirty(bits: DirtyGeometry);
365
366 setSubtitleGeometry();
367}
368
369const QVideoFrameTexturePoolPtr &QSGVideoNode::texturePool() const
370{
371 return m_material->m_texturePool;
372}
373
374QT_END_NAMESPACE
375

source code of qtmultimedia/src/multimediaquick/qsgvideonode_p.cpp