1 | // Copyright (C) 2021 The Qt Company Ltd. |
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | |
4 | #include "qsgvideonode_p.h" |
5 | #include <QtQuick/qsgmaterial.h> |
6 | #include "qsgvideotexture_p.h" |
7 | #include <QtMultimedia/private/qvideotexturehelper_p.h> |
8 | #include <private/qquicktextnode_p.h> |
9 | #include <private/qquickvideooutput_p.h> |
10 | #include <private/qabstractvideobuffer_p.h> |
11 | #include <qmutex.h> |
12 | |
13 | QT_BEGIN_NAMESPACE |
14 | |
15 | /* Helpers */ |
16 | static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p) |
17 | { |
18 | v->x = p.x(); |
19 | v->y = p.y(); |
20 | } |
21 | |
22 | static inline void qSetTex(QSGGeometry::TexturedPoint2D *v, const QPointF &p) |
23 | { |
24 | v->tx = p.x(); |
25 | v->ty = p.y(); |
26 | } |
27 | |
28 | static inline void qSwapTex(QSGGeometry::TexturedPoint2D *v0, QSGGeometry::TexturedPoint2D *v1) |
29 | { |
30 | auto tvx = v0->tx; |
31 | auto tvy = v0->ty; |
32 | v0->tx = v1->tx; |
33 | v0->ty = v1->ty; |
34 | v1->tx = tvx; |
35 | v1->ty = tvy; |
36 | } |
37 | |
38 | class QSGVideoMaterial; |
39 | |
40 | class QSGVideoMaterialRhiShader : public QSGMaterialShader |
41 | { |
42 | public: |
43 | QSGVideoMaterialRhiShader(const QVideoFrameFormat &format) |
44 | : m_format(format) |
45 | { |
46 | setShaderFileName(stage: VertexStage, filename: m_format.vertexShaderFileName()); |
47 | setShaderFileName(stage: FragmentStage, filename: m_format.fragmentShaderFileName()); |
48 | } |
49 | |
50 | bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, |
51 | QSGMaterial *oldMaterial) override; |
52 | |
53 | void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, |
54 | QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; |
55 | |
56 | protected: |
57 | QVideoFrameFormat m_format; |
58 | float m_planeWidth[3] = {0, 0, 0}; |
59 | QMatrix4x4 m_colorMatrix; |
60 | }; |
61 | |
62 | class QSGVideoMaterial : public QSGMaterial |
63 | { |
64 | public: |
65 | QSGVideoMaterial(const QVideoFrameFormat &format); |
66 | |
67 | [[nodiscard]] QSGMaterialType *type() const override { |
68 | static QSGMaterialType type[QVideoFrameFormat::NPixelFormats]; |
69 | return &type[m_format.pixelFormat()]; |
70 | } |
71 | |
72 | [[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override { |
73 | return new QSGVideoMaterialRhiShader(m_format); |
74 | } |
75 | |
76 | int compare(const QSGMaterial *other) const override { |
77 | const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other); |
78 | |
79 | qint64 diff = m_textures[0].comparisonKey() - m->m_textures[0].comparisonKey(); |
80 | if (!diff) |
81 | diff = m_textures[1].comparisonKey() - m->m_textures[1].comparisonKey(); |
82 | if (!diff) |
83 | diff = m_textures[2].comparisonKey() - m->m_textures[2].comparisonKey(); |
84 | |
85 | return diff < 0 ? -1 : (diff > 0 ? 1 : 0); |
86 | } |
87 | |
88 | void updateBlending() { |
89 | // ### respect video formats with Alpha |
90 | setFlag(flags: Blending, on: !qFuzzyCompare(p1: m_opacity, p2: float(1.0))); |
91 | } |
92 | |
93 | void setCurrentFrame(const QVideoFrame &frame) { |
94 | QMutexLocker lock(&m_frameMutex); |
95 | m_currentFrame = frame; |
96 | m_texturesDirty = true; |
97 | } |
98 | |
99 | void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates); |
100 | |
101 | QVideoFrameFormat m_format; |
102 | float m_planeWidth[3]; |
103 | float m_opacity; |
104 | |
105 | QMutex m_frameMutex; |
106 | bool m_texturesDirty = false; |
107 | QVideoFrame m_currentFrame; |
108 | |
109 | enum { NVideoFrameSlots = 4 }; |
110 | QVideoFrame m_videoFrameSlots[NVideoFrameSlots]; |
111 | std::array<QSGVideoTexture, 3> m_textures; |
112 | std::unique_ptr<QVideoFrameTextures> m_videoFrameTextures; |
113 | }; |
114 | |
115 | void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates) |
116 | { |
117 | QMutexLocker locker(&m_frameMutex); |
118 | if (!m_texturesDirty) |
119 | return; |
120 | |
121 | // keep the video frames alive until we know that they are not needed anymore |
122 | Q_ASSERT(NVideoFrameSlots >= rhi->resourceLimit(QRhi::FramesInFlight)); |
123 | m_videoFrameSlots[rhi->currentFrameSlot()] = m_currentFrame; |
124 | |
125 | // update and upload all textures |
126 | m_videoFrameTextures = QVideoTextureHelper::createTextures(frame&: m_currentFrame, rhi, rub: resourceUpdates, oldTextures: std::move(m_videoFrameTextures)); |
127 | if (!m_videoFrameTextures) |
128 | return; |
129 | |
130 | for (int plane = 0; plane < 3; ++plane) |
131 | m_textures[plane].setRhiTexture(m_videoFrameTextures->texture(plane)); |
132 | m_texturesDirty = false; |
133 | } |
134 | |
135 | |
136 | bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, |
137 | QSGMaterial *oldMaterial) |
138 | { |
139 | Q_UNUSED(oldMaterial); |
140 | |
141 | auto m = static_cast<QSGVideoMaterial *>(newMaterial); |
142 | |
143 | if (!state.isMatrixDirty() && !state.isOpacityDirty()) |
144 | return false; |
145 | |
146 | if (state.isOpacityDirty()) { |
147 | m->m_opacity = state.opacity(); |
148 | m->updateBlending(); |
149 | } |
150 | |
151 | // Do this here, not in updateSampledImage. First, with multiple textures we want to |
152 | // do this once. More importantly, on some platforms (Android) the externalMatrix is |
153 | // updated by this function and we need that already in updateUniformData. |
154 | m->updateTextures(rhi: state.rhi(), resourceUpdates: state.resourceUpdateBatch()); |
155 | |
156 | m_format.updateUniformData(dst: state.uniformData(), frame: m->m_currentFrame, |
157 | transform: state.combinedMatrix(), opacity: state.opacity()); |
158 | |
159 | return true; |
160 | } |
161 | |
162 | void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, |
163 | QSGMaterial *newMaterial, QSGMaterial *oldMaterial) |
164 | { |
165 | Q_UNUSED(state); |
166 | Q_UNUSED(oldMaterial); |
167 | if (binding < 1 || binding > 3) |
168 | return; |
169 | |
170 | auto m = static_cast<QSGVideoMaterial *>(newMaterial); |
171 | *texture = &m->m_textures[binding - 1]; |
172 | } |
173 | |
174 | QSGVideoMaterial::QSGVideoMaterial(const QVideoFrameFormat &format) : |
175 | m_format(format), |
176 | m_opacity(1.0) |
177 | { |
178 | setFlag(flags: Blending, on: false); |
179 | } |
180 | |
181 | QSGVideoNode::QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &format) |
182 | : m_parent(parent), |
183 | m_orientation(-1), |
184 | m_frameOrientation(-1), |
185 | m_frameMirrored(false), |
186 | m_format(format) |
187 | { |
188 | setFlag(QSGNode::OwnsMaterial); |
189 | setFlag(QSGNode::OwnsGeometry); |
190 | m_material = new QSGVideoMaterial(format); |
191 | setMaterial(m_material); |
192 | } |
193 | |
194 | QSGVideoNode::~QSGVideoNode() |
195 | { |
196 | delete m_subtitleTextNode; |
197 | } |
198 | |
199 | void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame) |
200 | { |
201 | m_material->setCurrentFrame(frame); |
202 | markDirty(bits: DirtyMaterial); |
203 | updateSubtitle(frame); |
204 | } |
205 | |
206 | void QSGVideoNode::updateSubtitle(const QVideoFrame &frame) |
207 | { |
208 | QSize subtitleFrameSize = m_rect.size().toSize(); |
209 | if (subtitleFrameSize.isEmpty()) |
210 | return; |
211 | if (m_orientation % 180) |
212 | subtitleFrameSize.transpose(); |
213 | if (!m_subtitleLayout.update(frameSize: subtitleFrameSize, text: frame.subtitleText())) |
214 | return; |
215 | |
216 | delete m_subtitleTextNode; |
217 | m_subtitleTextNode = nullptr; |
218 | if (frame.subtitleText().isEmpty()) |
219 | return; |
220 | |
221 | m_subtitleTextNode = new QQuickTextNode(m_parent); |
222 | QColor bgColor = Qt::black; |
223 | bgColor.setAlpha(128); |
224 | m_subtitleTextNode->addRectangleNode(rect: m_subtitleLayout.bounds, color: bgColor); |
225 | m_subtitleTextNode->addTextLayout(position: m_subtitleLayout.layout.position(), textLayout: &m_subtitleLayout.layout, color: Qt::white); |
226 | appendChildNode(node: m_subtitleTextNode); |
227 | setSubtitleGeometry(); |
228 | } |
229 | |
230 | void QSGVideoNode::setSubtitleGeometry() |
231 | { |
232 | if (!m_subtitleTextNode) |
233 | return; |
234 | |
235 | if (m_material) |
236 | updateSubtitle(frame: m_material->m_currentFrame); |
237 | |
238 | float rotate = -1.f * m_orientation; |
239 | float yTranslate = 0; |
240 | float xTranslate = 0; |
241 | if (m_orientation == 90) { |
242 | yTranslate = m_rect.height(); |
243 | } else if (m_orientation == 180) { |
244 | yTranslate = m_rect.height(); |
245 | xTranslate = m_rect.width(); |
246 | } else if (m_orientation == 270) { |
247 | xTranslate = m_rect.width(); |
248 | } |
249 | |
250 | QMatrix4x4 transform; |
251 | transform.translate(x: m_rect.x() + xTranslate, y: m_rect.y() + yTranslate); |
252 | transform.rotate(angle: rotate, x: 0, y: 0, z: 1); |
253 | |
254 | m_subtitleTextNode->setMatrix(transform); |
255 | m_subtitleTextNode->markDirty(bits: DirtyGeometry); |
256 | } |
257 | |
258 | /* Update the vertices and texture coordinates. Orientation must be in {0,90,180,270} */ |
259 | void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &textureRect, int orientation) |
260 | { |
261 | bool frameChanged = false; |
262 | if (m_material) { |
263 | if (m_material->m_currentFrame.rotationAngle() != m_frameOrientation |
264 | || m_material->m_currentFrame.mirrored() != m_frameMirrored) { |
265 | frameChanged = true; |
266 | } |
267 | } |
268 | if (rect == m_rect && textureRect == m_textureRect && orientation == m_orientation |
269 | && !frameChanged) |
270 | return; |
271 | |
272 | m_rect = rect; |
273 | m_textureRect = textureRect; |
274 | m_orientation = orientation; |
275 | if (m_material) { |
276 | m_frameOrientation = m_material->m_currentFrame.rotationAngle(); |
277 | m_frameMirrored = m_material->m_currentFrame.mirrored(); |
278 | } |
279 | int videoRotation = orientation; |
280 | videoRotation += m_material ? m_material->m_currentFrame.rotationAngle() : 0; |
281 | videoRotation %= 360; |
282 | |
283 | QSGGeometry *g = geometry(); |
284 | |
285 | if (g == nullptr) |
286 | g = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4); |
287 | |
288 | QSGGeometry::TexturedPoint2D *v = g->vertexDataAsTexturedPoint2D(); |
289 | |
290 | // Set geometry first |
291 | qSetGeom(v: v + 0, p: rect.topLeft()); |
292 | qSetGeom(v: v + 1, p: rect.bottomLeft()); |
293 | qSetGeom(v: v + 2, p: rect.topRight()); |
294 | qSetGeom(v: v + 3, p: rect.bottomRight()); |
295 | |
296 | // and then texture coordinates |
297 | switch (videoRotation) { |
298 | default: |
299 | // tl, bl, tr, br |
300 | qSetTex(v: v + 0, p: textureRect.topLeft()); |
301 | qSetTex(v: v + 1, p: textureRect.bottomLeft()); |
302 | qSetTex(v: v + 2, p: textureRect.topRight()); |
303 | qSetTex(v: v + 3, p: textureRect.bottomRight()); |
304 | break; |
305 | |
306 | case 90: |
307 | // bl, br, tl, tr |
308 | qSetTex(v: v + 0, p: textureRect.bottomLeft()); |
309 | qSetTex(v: v + 1, p: textureRect.bottomRight()); |
310 | qSetTex(v: v + 2, p: textureRect.topLeft()); |
311 | qSetTex(v: v + 3, p: textureRect.topRight()); |
312 | break; |
313 | |
314 | case 180: |
315 | // br, tr, bl, tl |
316 | qSetTex(v: v + 0, p: textureRect.bottomRight()); |
317 | qSetTex(v: v + 1, p: textureRect.topRight()); |
318 | qSetTex(v: v + 2, p: textureRect.bottomLeft()); |
319 | qSetTex(v: v + 3, p: textureRect.topLeft()); |
320 | break; |
321 | |
322 | case 270: |
323 | // tr, tl, br, bl |
324 | qSetTex(v: v + 0, p: textureRect.topRight()); |
325 | qSetTex(v: v + 1, p: textureRect.topLeft()); |
326 | qSetTex(v: v + 2, p: textureRect.bottomRight()); |
327 | qSetTex(v: v + 3, p: textureRect.bottomLeft()); |
328 | break; |
329 | } |
330 | |
331 | if (m_material && m_material->m_currentFrame.mirrored()) { |
332 | qSwapTex(v0: v + 0, v1: v + 2); |
333 | qSwapTex(v0: v + 1, v1: v + 3); |
334 | } |
335 | |
336 | if (!geometry()) |
337 | setGeometry(g); |
338 | |
339 | markDirty(bits: DirtyGeometry); |
340 | |
341 | setSubtitleGeometry(); |
342 | } |
343 | |
344 | QT_END_NAMESPACE |
345 | |