1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstvideobuffer_p.h"
5#include "qgstreamervideosink_p.h"
6#include <private/qvideotexturehelper_p.h>
7#include <qpa/qplatformnativeinterface.h>
8#include <qguiapplication.h>
9#include <QtCore/qloggingcategory.h>
10
11#include <gst/video/video.h>
12#include <gst/video/video-frame.h>
13#include <gst/video/gstvideometa.h>
14#include <gst/pbutils/gstpluginsbaseversion.h>
15
16#include <common/qgstutils_p.h>
17
18#if QT_CONFIG(gstreamer_gl)
19# include <QtGui/rhi/qrhi.h>
20# include <QtGui/qopenglcontext.h>
21# include <QtGui/qopenglfunctions.h>
22# include <QtGui/qopengl.h>
23
24# include <gst/gl/gstglconfig.h>
25# include <gst/gl/gstglmemory.h>
26# include <gst/gl/gstglsyncmeta.h>
27
28# if QT_CONFIG(gstreamer_gl_egl)
29# include <EGL/egl.h>
30# include <EGL/eglext.h>
31# endif
32
33# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
34# include <gst/allocators/gstdmabuf.h>
35# endif
36#endif
37
38QT_BEGIN_NAMESPACE
39
40Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer, "qt.multimedia.gstreamer.videobuffer");
41
42// keep things building without drm_fourcc.h
43#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | \
44 ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
45
46#define DRM_FORMAT_RGBA8888 fourcc_code('R', 'A', '2', '4') /* [31:0] R:G:B:A 8:8:8:8 little endian */
47#define DRM_FORMAT_RGB888 fourcc_code('R', 'G', '2', '4') /* [23:0] R:G:B little endian */
48#define DRM_FORMAT_RG88 fourcc_code('R', 'G', '8', '8') /* [15:0] R:G 8:8 little endian */
49#define DRM_FORMAT_ABGR8888 fourcc_code('A', 'B', '2', '4') /* [31:0] A:B:G:R 8:8:8:8 little endian */
50#define DRM_FORMAT_BGR888 fourcc_code('B', 'G', '2', '4') /* [23:0] B:G:R little endian */
51#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8') /* [15:0] G:R 8:8 little endian */
52#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ') /* [7:0] R */
53#define DRM_FORMAT_R16 fourcc_code('R', '1', '6', ' ') /* [15:0] R little endian */
54#define DRM_FORMAT_RGB565 fourcc_code('R', 'G', '1', '6') /* [15:0] R:G:B 5:6:5 little endian */
55#define DRM_FORMAT_RG1616 fourcc_code('R', 'G', '3', '2') /* [31:0] R:G 16:16 little endian */
56#define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */
57#define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */
58
59QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info,
60 QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat,
61 QGstCaps::MemoryFormat format)
62 : QHwVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory)
63 ? QVideoFrame::RhiTextureHandle
64 : QVideoFrame::NoHandle,
65 sink ? sink->rhi() : nullptr),
66 m_memoryFormat(format),
67 m_frameFormat(frameFormat),
68 m_rhi(sink ? sink->rhi() : nullptr),
69 m_videoInfo(info),
70 m_buffer(std::move(buffer))
71{
72#if QT_CONFIG(gstreamer_gl_egl)
73 if (sink) {
74 eglDisplay = sink->eglDisplay();
75 eglImageTargetTexture2D = sink->eglImageTargetTexture2D();
76 }
77#endif
78 Q_UNUSED(m_memoryFormat);
79 Q_UNUSED(eglDisplay);
80 Q_UNUSED(eglImageTargetTexture2D);
81}
82
83QGstVideoBuffer::~QGstVideoBuffer()
84{
85 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
86}
87
88QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode)
89{
90 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
91 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
92
93 MapData mapData;
94 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
95 return mapData;
96
97 if (m_videoInfo.finfo->n_planes == 0) { // Encoded
98 if (gst_buffer_map(buffer: m_buffer.get(), info: &m_frame.map[0], flags)) {
99 mapData.planeCount = 1;
100 mapData.bytesPerLine[0] = -1;
101 mapData.dataSize[0] = m_frame.map[0].size;
102 mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data);
103
104 m_mode = mode;
105 }
106 } else if (gst_video_frame_map(frame: &m_frame, info: &m_videoInfo, buffer: m_buffer.get(), flags)) {
107 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
108
109 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
110 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
111 mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
112 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
113 }
114
115 m_mode = mode;
116 }
117 return mapData;
118}
119
120void QGstVideoBuffer::unmap()
121{
122 if (m_mode != QVideoFrame::NotMapped) {
123 if (m_videoInfo.finfo->n_planes == 0)
124 gst_buffer_unmap(buffer: m_buffer.get(), info: &m_frame.map[0]);
125 else
126 gst_video_frame_unmap(frame: &m_frame);
127 }
128 m_mode = QVideoFrame::NotMapped;
129}
130
131#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
132static int
133fourccFromVideoInfo(const GstVideoInfo * info, int plane)
134{
135 GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info);
136#if G_BYTE_ORDER == G_LITTLE_ENDIAN
137 const gint rgba_fourcc = DRM_FORMAT_ABGR8888;
138 const gint rgb_fourcc = DRM_FORMAT_BGR888;
139 const gint rg_fourcc = DRM_FORMAT_GR88;
140#else
141 const gint rgba_fourcc = DRM_FORMAT_RGBA8888;
142 const gint rgb_fourcc = DRM_FORMAT_RGB888;
143 const gint rg_fourcc = DRM_FORMAT_RG88;
144#endif
145
146 GST_DEBUG ("Getting DRM fourcc for %s plane %i",
147 gst_video_format_to_string (format), plane);
148
149 switch (format) {
150 case GST_VIDEO_FORMAT_RGB16:
151 case GST_VIDEO_FORMAT_BGR16:
152 return DRM_FORMAT_RGB565;
153
154 case GST_VIDEO_FORMAT_RGB:
155 case GST_VIDEO_FORMAT_BGR:
156 return rgb_fourcc;
157
158 case GST_VIDEO_FORMAT_RGBA:
159 case GST_VIDEO_FORMAT_RGBx:
160 case GST_VIDEO_FORMAT_BGRA:
161 case GST_VIDEO_FORMAT_BGRx:
162 case GST_VIDEO_FORMAT_ARGB:
163 case GST_VIDEO_FORMAT_xRGB:
164 case GST_VIDEO_FORMAT_ABGR:
165 case GST_VIDEO_FORMAT_xBGR:
166 case GST_VIDEO_FORMAT_AYUV:
167#if GST_CHECK_PLUGINS_BASE_VERSION(1,16,0)
168 case GST_VIDEO_FORMAT_VUYA:
169#endif
170 return rgba_fourcc;
171
172 case GST_VIDEO_FORMAT_GRAY8:
173 return DRM_FORMAT_R8;
174
175 case GST_VIDEO_FORMAT_YUY2:
176 case GST_VIDEO_FORMAT_UYVY:
177 case GST_VIDEO_FORMAT_GRAY16_LE:
178 case GST_VIDEO_FORMAT_GRAY16_BE:
179 return rg_fourcc;
180
181 case GST_VIDEO_FORMAT_NV12:
182 case GST_VIDEO_FORMAT_NV21:
183 return plane == 0 ? DRM_FORMAT_R8 : rg_fourcc;
184
185 case GST_VIDEO_FORMAT_I420:
186 case GST_VIDEO_FORMAT_YV12:
187 case GST_VIDEO_FORMAT_Y41B:
188 case GST_VIDEO_FORMAT_Y42B:
189 case GST_VIDEO_FORMAT_Y444:
190 return DRM_FORMAT_R8;
191
192#if GST_CHECK_PLUGINS_BASE_VERSION(1,16,0)
193 case GST_VIDEO_FORMAT_BGR10A2_LE:
194 return DRM_FORMAT_BGRA1010102;
195#endif
196
197// case GST_VIDEO_FORMAT_RGB10A2_LE:
198// return DRM_FORMAT_RGBA1010102;
199
200 case GST_VIDEO_FORMAT_P010_10LE:
201// case GST_VIDEO_FORMAT_P012_LE:
202// case GST_VIDEO_FORMAT_P016_LE:
203 return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_GR1616;
204
205 case GST_VIDEO_FORMAT_P010_10BE:
206// case GST_VIDEO_FORMAT_P012_BE:
207// case GST_VIDEO_FORMAT_P016_BE:
208 return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_RG1616;
209
210 default:
211 GST_ERROR ("Unsupported format for DMABuf.");
212 return -1;
213 }
214}
215#endif
216
217#if QT_CONFIG(gstreamer_gl)
218struct GlTextures
219{
220 uint count = 0;
221 bool owned = false;
222 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
223};
224
225class QGstQVideoFrameTextures : public QVideoFrameTextures
226{
227public:
228 QGstQVideoFrameTextures(QRhi *rhi, QSize size, QVideoFrameFormat::PixelFormat format, GlTextures &textures)
229 : m_rhi(rhi)
230 , m_glTextures(textures)
231 {
232 auto desc = QVideoTextureHelper::textureDescription(format);
233 for (uint i = 0; i < textures.count; ++i) {
234 QSize planeSize = desc->rhiPlaneSize(frameSize: size, plane: i, rhi);
235 m_textures[i].reset(p: rhi->newTexture(format: desc->rhiTextureFormat(plane: i, rhi: m_rhi), pixelSize: planeSize, sampleCount: 1, flags: {}));
236 m_textures[i]->createFrom(src: {.object: textures.names[i], .layout: 0});
237 }
238 }
239
240 ~QGstQVideoFrameTextures() override
241 {
242 m_rhi->makeThreadLocalNativeContextCurrent();
243 auto ctx = QOpenGLContext::currentContext();
244 if (m_glTextures.owned && ctx)
245 ctx->functions()->glDeleteTextures(n: int(m_glTextures.count), textures: m_glTextures.names.data());
246 }
247
248 QRhiTexture *texture(uint plane) const override
249 {
250 return plane < m_glTextures.count ? m_textures[plane].get() : nullptr;
251 }
252
253private:
254 QRhi *m_rhi = nullptr;
255 GlTextures m_glTextures;
256 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
257};
258
259static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
260 GstVideoInfo &videoInfo)
261{
262 qCDebug(qLcGstVideoBuffer) << "mapFromGlTexture";
263
264 GstBuffer *buffer = bufferHandle.get();
265 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
266 if (!mem)
267 return {};
268
269 if (!gst_video_frame_map(frame: &frame, info: &videoInfo, buffer, flags: GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
270 qWarning() << "Could not map GL textures";
271 return {};
272 }
273
274 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
275 GstBuffer *sync_buffer = nullptr;
276 if (!sync_meta) {
277 sync_buffer = gst_buffer_new();
278 sync_meta = gst_buffer_add_gl_sync_meta(context: mem->context, buffer: sync_buffer);
279 }
280 gst_gl_sync_meta_set_sync_point (sync_meta, context: mem->context);
281 gst_gl_sync_meta_wait (sync_meta, context: mem->context);
282 if (sync_buffer)
283 gst_buffer_unref(buf: sync_buffer);
284
285 GlTextures textures;
286 textures.count = frame.info.finfo->n_planes;
287
288 for (uint i = 0; i < textures.count; ++i)
289 textures.names[i] = *(guint32 *)frame.data[i];
290
291 gst_video_frame_unmap(frame: &frame);
292
293 return textures;
294}
295
296# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
297static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle,
298 GstVideoFrame &frame, GstVideoInfo &videoInfo,
299 Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D)
300{
301 qCDebug(qLcGstVideoBuffer) << "mapFromDmaBuffer";
302
303 GstBuffer *buffer = bufferHandle.get();
304
305 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
306 Q_ASSERT(eglDisplay);
307 Q_ASSERT(eglImageTargetTexture2D);
308
309 auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
310 auto glContext = nativeHandles->context;
311 if (!glContext) {
312 qWarning() << "no GL context";
313 return {};
314 }
315
316 if (!gst_video_frame_map(frame: &frame, info: &videoInfo, buffer, flags: GstMapFlags(GST_MAP_READ))) {
317 qDebug() << "Couldn't map DMA video frame";
318 return {};
319 }
320
321 GlTextures textures = {};
322 textures.owned = true;
323 textures.count = GST_VIDEO_FRAME_N_PLANES(&frame);
324 // int width = GST_VIDEO_FRAME_WIDTH(&frame);
325 // int height = GST_VIDEO_FRAME_HEIGHT(&frame);
326 if (textures.count != gst_buffer_n_memory(buffer)) {
327 qCDebug(qLcGstVideoBuffer) << "mapFromDmaBuffer: Unsupported memory layout, creating "
328 "textures from system memory instead. Will be fixed by "
329 "https://codereview.qt-project.org/c/qt/qtmultimedia/+/662143";
330 return {};
331 }
332 Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&frame) == gst_buffer_n_memory(buffer));
333
334 QOpenGLFunctions functions(glContext);
335 functions.glGenTextures(n: int(textures.count), textures: textures.names.data());
336
337 // qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError();
338 // qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2];
339
340 for (int i = 0; i < int(textures.count); ++i) {
341 auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&frame, i);
342 auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, i);
343 int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&frame, i);
344 int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&frame, i);
345 auto mem = gst_buffer_peek_memory(buffer, idx: i);
346 int fd = gst_dmabuf_memory_get_fd(mem);
347
348 // qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd;
349 // ### do we need to open/close the fd?
350 // ### can we convert several planes at once?
351 // Get the correct DRM_FORMATs from the texture format in the description
352 EGLAttrib const attribute_list[] = {
353 EGL_WIDTH, planeWidth,
354 EGL_HEIGHT, planeHeight,
355 EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(info: &videoInfo, plane: i),
356 EGL_DMA_BUF_PLANE0_FD_EXT, fd,
357 EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset,
358 EGL_DMA_BUF_PLANE0_PITCH_EXT, stride,
359 EGL_NONE
360 };
361 EGLImage image = eglCreateImage(dpy: eglDisplay,
362 EGL_NO_CONTEXT,
363 EGL_LINUX_DMA_BUF_EXT,
364 buffer: nullptr,
365 attrib_list: attribute_list);
366 if (image == EGL_NO_IMAGE_KHR) {
367 qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError();
368 }
369 // qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError();
370 functions.glBindTexture(GL_TEXTURE_2D, texture: textures.names[i]);
371 // qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError();
372 auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D;
373 EGLImageTargetTexture2D(GL_TEXTURE_2D, image);
374 // qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError();
375 eglDestroyImage(dpy: eglDisplay, image);
376 }
377 gst_video_frame_unmap(frame: &frame);
378
379 return textures;
380}
381#endif
382#endif
383
384QVideoFrameTexturesUPtr QGstVideoBuffer::mapTextures(QRhi &rhi, QVideoFrameTexturesUPtr& /*oldTextures*/)
385{
386#if QT_CONFIG(gstreamer_gl)
387 GlTextures textures = {};
388 if (m_memoryFormat == QGstCaps::GLTexture)
389 textures = mapFromGlTexture(bufferHandle: m_buffer, frame&: m_frame, videoInfo&: m_videoInfo);
390
391# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
392 else if (m_memoryFormat == QGstCaps::DMABuf)
393 textures = mapFromDmaBuffer(rhi: m_rhi, bufferHandle: m_buffer, frame&: m_frame, videoInfo&: m_videoInfo, eglDisplay,
394 eglImageTargetTexture2D);
395
396# endif
397 if (textures.count > 0)
398 return std::make_unique<QGstQVideoFrameTextures>(args: &rhi, args: QSize{m_videoInfo.width, m_videoInfo.height},
399 args: m_frameFormat.pixelFormat(), args&: textures);
400#endif
401 return {};
402}
403
404QT_END_NAMESPACE
405

source code of qtmultimedia/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp