1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <common/qgstreamervideosink_p.h>
5#include <common/qgstvideorenderersink_p.h>
6#include <common/qgst_debug_p.h>
7#include <common/qgstutils_p.h>
8#include <rhi/qrhi.h>
9
10#include <QtCore/qdebug.h>
11#include <QtCore/qloggingcategory.h>
12
13#if QT_CONFIG(gstreamer_gl)
14# include <QtGui/qguiapplication.h>
15# include <QtGui/qopenglcontext.h>
16# include <QtGui/qwindow.h>
17# include <QtGui/qpa/qplatformnativeinterface.h>
18# include <gst/gl/gstglconfig.h>
19# include <gst/gl/gstgldisplay.h>
20
21# if QT_CONFIG(gstreamer_gl_x11)
22# include <gst/gl/x11/gstgldisplay_x11.h>
23# endif
24# if QT_CONFIG(gstreamer_gl_egl)
25# include <gst/gl/egl/gstgldisplay_egl.h>
26# include <EGL/egl.h>
27# include <EGL/eglext.h>
28# endif
29# if QT_CONFIG(gstreamer_gl_wayland)
30# include <gst/gl/wayland/gstgldisplay_wayland.h>
31# endif
32#endif // #if QT_CONFIG(gstreamer_gl)
33
34QT_BEGIN_NAMESPACE
35
36static Q_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink");
37
38QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent)
39 : QPlatformVideoSink{
40 parent,
41 },
42 m_sinkBin{
43 QGstBin::create(name: "videoSinkBin"),
44 }
45{
46 // This is a hack for some iMX and NVidia platforms. These require the use of a special video
47 // conversion element in the pipeline before the video sink, as they unfortunately
48 // output some proprietary format from the decoder even though it's sometimes marked as
49 // a regular supported video/x-raw format.
50 //
51 // To fix this, simply insert the element into the pipeline if it's available. Otherwise
52 // we simply use an identity element.
53 QGstElementFactoryHandle factory;
54
55 // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the
56 // conversion element. Ideally we construct the element programatically, though.
57 QByteArray preprocessOverride = qgetenv(varName: "QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT");
58 if (!preprocessOverride.isEmpty()) {
59 qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:"
60 << preprocessOverride;
61
62 m_gstPreprocess = QGstBin::createFromPipelineDescription(pipelineDescription: preprocessOverride, name: nullptr,
63 /*ghostUnlinkedPads=*/true);
64 if (!m_gstPreprocess)
65 qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride;
66 }
67
68 if (!m_gstPreprocess) {
69 // This is a hack for some iMX and NVidia platforms. These require the use of a special
70 // video conversion element in the pipeline before the video sink, as they unfortunately
71 // output some proprietary format from the decoder even though it's sometimes marked as
72 // a regular supported video/x-raw format.
73 static constexpr auto decodersToTest = {
74 "imxvideoconvert_g2d",
75 "nvvidconv",
76 };
77
78 for (const char *decoder : decodersToTest) {
79 factory = QGstElement::findFactory(decoder);
80 if (factory)
81 break;
82 }
83
84 if (factory) {
85 qCDebug(qLcGstVideoSink)
86 << "instantiating conversion element:"
87 << g_type_name(type: gst_element_factory_get_element_type(factory: factory.get()));
88
89 m_gstPreprocess = QGstElement::createFromFactory(factory, name: "preprocess");
90 }
91 }
92
93 bool disablePixelAspectRatio =
94 qEnvironmentVariableIsSet(varName: "QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO");
95 if (disablePixelAspectRatio) {
96 // Enabling the pixel aspect ratio may expose a gstreamer bug on cameras that don't expose a
97 // pixel-aspect-ratio via `VIDIOC_CROPCAP`. This can cause the caps negotiation to fail.
98 // Using the QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO environment variable, one can disable
99 // pixel-aspect-ratio handling
100 //
101 // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242
102 m_gstCapsFilter =
103 QGstElement::createFromFactory(factory: "identity", name: "nullPixelAspectRatioCapsFilter");
104 } else {
105 m_gstCapsFilter =
106 QGstElement::createFromFactory(factory: "capsfilter", name: "pixelAspectRatioCapsFilter");
107 QGstCaps capsFilterCaps{
108 gst_caps_new_simple(media_type: "video/x-raw", fieldname: "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL),
109 QGstCaps::HasRef,
110 };
111 g_object_set(object: m_gstCapsFilter.element(), first_property_name: "caps", capsFilterCaps.caps(), NULL);
112 }
113
114 if (m_gstPreprocess) {
115 m_sinkBin.add(ts: m_gstPreprocess, ts: m_gstCapsFilter);
116 qLinkGstElements(ts: m_gstPreprocess, ts: m_gstCapsFilter);
117 m_sinkBin.addGhostPad(child: m_gstPreprocess, name: "sink");
118 } else {
119 m_sinkBin.add(ts: m_gstCapsFilter);
120 m_sinkBin.addGhostPad(child: m_gstCapsFilter, name: "sink");
121 }
122}
123
124QGstreamerVideoSink::~QGstreamerVideoSink()
125{
126 emit aboutToBeDestroyed();
127
128 unrefGstContexts();
129}
130
131QGstElement QGstreamerVideoSink::gstSink()
132{
133 if (!m_gstVideoSink) {
134 if (!m_gstQtSink)
135 createQtSink();
136
137 updateSinkElement(newSink: m_gstQtSink);
138 }
139
140 return m_sinkBin;
141}
142
143void QGstreamerVideoSink::setActive(bool isActive)
144{
145 if (m_isActive == isActive)
146 return;
147 m_isActive = isActive;
148
149 if (m_gstQtSink)
150 m_gstQtSink.setActive(isActive);
151}
152
153void QGstreamerVideoSink::setAsync(bool isAsync)
154{
155 m_sinkIsAsync = isAsync;
156}
157
158void QGstreamerVideoSink::setRhi(QRhi *rhi)
159{
160 if (rhi && rhi->backend() != QRhi::OpenGLES2)
161 rhi = nullptr;
162 if (m_rhi == rhi)
163 return;
164
165 m_rhi = rhi;
166 updateGstContexts();
167 if (m_gstQtSink) {
168 QGstVideoRendererSinkElement oldSink = std::move(m_gstQtSink);
169
170 // force creation of a new sink with proper caps.
171 createQtSink();
172 updateSinkElement(newSink: m_gstQtSink);
173 }
174}
175
176void QGstreamerVideoSink::createQtSink()
177{
178 Q_ASSERT(!m_gstQtSink);
179
180 m_gstQtSink = QGstVideoRendererSink::createSink(surface: this);
181 if (!m_sinkIsAsync)
182 m_gstQtSink.set(property: "async", b: false);
183 m_gstQtSink.setActive(m_isActive);
184}
185
186void QGstreamerVideoSink::updateSinkElement(QGstVideoRendererSinkElement newSink)
187{
188 if (newSink == m_gstVideoSink)
189 return;
190
191 m_gstCapsFilter.src().modifyPipelineInIdleProbe(f: [&] {
192 if (m_gstVideoSink)
193 m_sinkBin.stopAndRemoveElements(ts&: m_gstVideoSink);
194
195 m_gstVideoSink = std::move(newSink);
196 m_sinkBin.add(ts: m_gstVideoSink);
197 qLinkGstElements(ts: m_gstCapsFilter, ts: m_gstVideoSink);
198 GstEvent *event = gst_event_new_reconfigure();
199 gst_element_send_event(element: m_gstVideoSink.element(), event);
200 m_gstVideoSink.syncStateWithParent();
201 });
202
203 m_sinkBin.dumpPipelineGraph(filename: "updateSinkElement");
204}
205
206void QGstreamerVideoSink::unrefGstContexts()
207{
208 m_gstGlDisplayContext.reset();
209 m_gstGlLocalContext.reset();
210 m_eglDisplay = nullptr;
211 m_eglImageTargetTexture2D = nullptr;
212}
213
214void QGstreamerVideoSink::updateGstContexts()
215{
216 using namespace Qt::Literals;
217
218 unrefGstContexts();
219
220#if QT_CONFIG(gstreamer_gl)
221 if (!m_rhi || m_rhi->backend() != QRhi::OpenGLES2)
222 return;
223
224 auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(m_rhi->nativeHandles());
225 auto glContext = nativeHandles->context;
226 Q_ASSERT(glContext);
227
228 const QString platform = QGuiApplication::platformName();
229 QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface();
230 m_eglDisplay = pni->nativeResourceForIntegration(resource: "egldisplay"_ba);
231// qDebug() << "platform is" << platform << m_eglDisplay;
232
233 QGstGLDisplayHandle gstGlDisplay;
234
235 QByteArray contextName = "eglcontext"_ba;
236 GstGLPlatform glPlatform = GST_GL_PLATFORM_EGL;
237 // use the egl display if we have one
238 if (m_eglDisplay) {
239# if QT_CONFIG(gstreamer_gl_egl)
240 gstGlDisplay.reset(
241 GST_GL_DISPLAY_CAST(gst_gl_display_egl_new_with_egl_display(m_eglDisplay)),
242 mode: QGstGLDisplayHandle::HasRef);
243 m_eglImageTargetTexture2D = eglGetProcAddress(procname: "glEGLImageTargetTexture2DOES");
244# endif
245 } else {
246 auto display = pni->nativeResourceForIntegration(resource: "display"_ba);
247
248 if (display) {
249# if QT_CONFIG(gstreamer_gl_x11)
250 if (platform == QLatin1String("xcb")) {
251 contextName = "glxcontext"_ba;
252 glPlatform = GST_GL_PLATFORM_GLX;
253
254 gstGlDisplay.reset(GST_GL_DISPLAY_CAST(gst_gl_display_x11_new_with_display(
255 reinterpret_cast<Display *>(display))),
256 mode: QGstGLDisplayHandle::HasRef);
257 }
258# endif
259# if QT_CONFIG(gstreamer_gl_wayland)
260 if (platform.startsWith(s: QLatin1String("wayland"))) {
261 Q_ASSERT(!gstGlDisplay);
262 gstGlDisplay.reset(GST_GL_DISPLAY_CAST(gst_gl_display_wayland_new_with_display(
263 reinterpret_cast<struct wl_display *>(display))),
264 mode: QGstGLDisplayHandle::HasRef);
265 }
266#endif
267 }
268 }
269
270 if (!gstGlDisplay) {
271 qWarning() << "Could not create GstGLDisplay";
272 return;
273 }
274
275 void *nativeContext = pni->nativeResourceForContext(resource: contextName, context: glContext);
276 if (!nativeContext)
277 qWarning() << "Could not find resource for" << contextName;
278
279 GstGLAPI glApi = QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL ? GST_GL_API_OPENGL : GST_GL_API_GLES2;
280 QGstGLContextHandle appContext{
281 gst_gl_context_new_wrapped(display: gstGlDisplay.get(), handle: guintptr(nativeContext), context_type: glPlatform, available_apis: glApi),
282 QGstGLContextHandle::HasRef,
283 };
284 if (!appContext)
285 qWarning() << "Could not create wrappped context for platform:" << glPlatform;
286
287 gst_gl_context_activate(context: appContext.get(), activate: true);
288
289 QUniqueGErrorHandle error;
290 gst_gl_context_fill_info(context: appContext.get(), error: &error);
291 if (error) {
292 qWarning() << "Could not fill context info:" << error;
293 error = {};
294 }
295
296 QGstGLContextHandle displayContext;
297 gst_gl_display_create_context(display: gstGlDisplay.get(), other_context: appContext.get(), p_context: &displayContext, error: &error);
298 if (error)
299 qWarning() << "Could not create display context:" << error;
300
301 appContext.reset();
302
303 m_gstGlDisplayContext.reset(o: gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, persistent: false),
304 mode: QGstContextHandle::HasRef);
305 gst_context_set_gl_display(context: m_gstGlDisplayContext.get(), display: gstGlDisplay.get());
306
307 m_gstGlLocalContext.reset(o: gst_context_new(context_type: "gst.gl.local_context", persistent: false),
308 mode: QGstContextHandle::HasRef);
309 GstStructure *structure = gst_context_writable_structure(context: m_gstGlLocalContext.get());
310 gst_structure_set(structure, fieldname: "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr);
311 displayContext.reset();
312
313 // Note: after updating the context, we switch the sink and send gst_event_new_reconfigure()
314 // upstream. this will cause the context to be queried again.
315#endif // #if QT_CONFIG(gstreamer_gl)
316}
317
318QT_END_NAMESPACE
319
320#include "moc_qgstreamervideosink_p.cpp"
321

source code of qtmultimedia/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp