1// Copyright (C) 2020 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3//#define DEBUG_DECODER
4
5#include "qgstreameraudiodecoder_p.h"
6#include "qgstreamermessage_p.h"
7
8#include <qgstutils_p.h>
9
10#include <gst/gstvalue.h>
11#include <gst/base/gstbasesrc.h>
12
13#include <QtCore/qdatetime.h>
14#include <QtCore/qdebug.h>
15#include <QtCore/qsize.h>
16#include <QtCore/qtimer.h>
17#include <QtCore/qdebug.h>
18#include <QtCore/qdir.h>
19#include <QtCore/qstandardpaths.h>
20#include <QtCore/qurl.h>
21
22#define MAX_BUFFERS_IN_QUEUE 4
23
24QT_BEGIN_NAMESPACE
25
26typedef enum {
27 GST_PLAY_FLAG_VIDEO = 0x00000001,
28 GST_PLAY_FLAG_AUDIO = 0x00000002,
29 GST_PLAY_FLAG_TEXT = 0x00000004,
30 GST_PLAY_FLAG_VIS = 0x00000008,
31 GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010,
32 GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020,
33 GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040,
34 GST_PLAY_FLAG_DOWNLOAD = 0x00000080,
35 GST_PLAY_FLAG_BUFFERING = 0x000000100
36} GstPlayFlags;
37
38
39QMaybe<QPlatformAudioDecoder *> QGstreamerAudioDecoder::create(QAudioDecoder *parent)
40{
41 QGstElement audioconvert("audioconvert", "audioconvert");
42 if (!audioconvert)
43 return errorMessageCannotFindElement(element: "audioconvert");
44
45 QGstPipeline playbin = GST_PIPELINE_CAST(QGstElement("playbin", "playbin").element());
46 if (!playbin)
47 return errorMessageCannotFindElement(element: "playbin");
48
49 return new QGstreamerAudioDecoder(playbin, audioconvert, parent);
50}
51
52QGstreamerAudioDecoder::QGstreamerAudioDecoder(QGstPipeline playbin, QGstElement audioconvert,
53 QAudioDecoder *parent)
54 : QPlatformAudioDecoder(parent), m_playbin(playbin), m_audioConvert(audioconvert)
55{
56 // Sort out messages
57 m_playbin.installMessageFilter(filter: this);
58
59 // Set the rest of the pipeline up
60 setAudioFlags(true);
61
62 m_outputBin = QGstBin("audio-output-bin");
63 m_outputBin.add(element: m_audioConvert);
64
65 // add ghostpad
66 m_outputBin.addGhostPad(child: m_audioConvert, name: "sink");
67
68 g_object_set(object: m_playbin.object(), first_property_name: "audio-sink", m_outputBin.element(), NULL);
69 g_signal_connect(m_playbin.object(), "deep-notify::source",
70 (GCallback)&QGstreamerAudioDecoder::configureAppSrcElement, (gpointer)this);
71
72 // Set volume to 100%
73 gdouble volume = 1.0;
74 m_playbin.set(property: "volume", d: volume);
75}
76
77QGstreamerAudioDecoder::~QGstreamerAudioDecoder()
78{
79 stop();
80
81#if QT_CONFIG(gstreamer_app)
82 delete m_appSrc;
83#endif
84}
85
86#if QT_CONFIG(gstreamer_app)
87void QGstreamerAudioDecoder::configureAppSrcElement(GObject* object, GObject *orig, GParamSpec *pspec, QGstreamerAudioDecoder *self)
88{
89 Q_UNUSED(object);
90 Q_UNUSED(pspec);
91
92 // In case we switch from appsrc to not
93 if (!self->appsrc())
94 return;
95
96 GstElement *appsrc;
97 g_object_get(object: orig, first_property_name: "source", &appsrc, NULL);
98
99 auto *qAppSrc = self->appsrc();
100 qAppSrc->setExternalAppSrc(appsrc);
101 qAppSrc->setup(stream: self->mDevice);
102
103 g_object_unref(G_OBJECT(appsrc));
104}
105#endif
106
107bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message)
108{
109 GstMessage* gm = message.rawMessage();
110 if (gm) {
111 if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_DURATION) {
112 updateDuration();
113 } else if (GST_MESSAGE_SRC(gm) == m_playbin.object()) {
114 switch (GST_MESSAGE_TYPE(gm)) {
115 case GST_MESSAGE_STATE_CHANGED:
116 {
117 GstState oldState;
118 GstState newState;
119 GstState pending;
120
121 gst_message_parse_state_changed(message: gm, oldstate: &oldState, newstate: &newState, pending: &pending);
122
123#ifdef DEBUG_DECODER
124 QStringList states;
125 states << "GST_STATE_VOID_PENDING" << "GST_STATE_NULL" << "GST_STATE_READY" << "GST_STATE_PAUSED" << "GST_STATE_PLAYING";
126
127 qDebug() << QString("state changed: old: %1 new: %2 pending: %3") \
128 .arg(states[oldState]) \
129 .arg(states[newState]) \
130 .arg(states[pending]) << "internal" << m_state;
131#endif
132
133 bool isDecoding = false;
134 switch (newState) {
135 case GST_STATE_VOID_PENDING:
136 case GST_STATE_NULL:
137 case GST_STATE_READY:
138 break;
139 case GST_STATE_PLAYING:
140 isDecoding = true;
141 break;
142 case GST_STATE_PAUSED:
143 isDecoding = true;
144
145 //gstreamer doesn't give a reliable indication the duration
146 //information is ready, GST_MESSAGE_DURATION is not sent by most elements
147 //the duration is queried up to 5 times with increasing delay
148 m_durationQueries = 5;
149 updateDuration();
150 break;
151 }
152
153 setIsDecoding(isDecoding);
154 }
155 break;
156
157 case GST_MESSAGE_EOS:
158 m_playbin.setState(GST_STATE_NULL);
159 finished();
160 break;
161
162 case GST_MESSAGE_ERROR: {
163 GError *err;
164 gchar *debug;
165 gst_message_parse_error(message: gm, gerror: &err, debug: &debug);
166 if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND)
167 processInvalidMedia(errorCode: QAudioDecoder::FormatError, errorString: tr(s: "Cannot play stream of type: <unknown>"));
168 else
169 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: QString::fromUtf8(utf8: err->message));
170 qWarning() << "Error:" << QString::fromUtf8(utf8: err->message);
171 g_error_free(error: err);
172 g_free(mem: debug);
173 }
174 break;
175 case GST_MESSAGE_WARNING:
176 {
177 GError *err;
178 gchar *debug;
179 gst_message_parse_warning (message: gm, gerror: &err, debug: &debug);
180 qWarning() << "Warning:" << QString::fromUtf8(utf8: err->message);
181 g_error_free (error: err);
182 g_free (mem: debug);
183 }
184 break;
185#ifdef DEBUG_DECODER
186 case GST_MESSAGE_INFO:
187 {
188 GError *err;
189 gchar *debug;
190 gst_message_parse_info (gm, &err, &debug);
191 qDebug() << "Info:" << QString::fromUtf8(err->message);
192 g_error_free (err);
193 g_free (debug);
194 }
195 break;
196#endif
197 default:
198 break;
199 }
200 } else if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_ERROR) {
201 GError *err;
202 gchar *debug;
203 gst_message_parse_error(message: gm, gerror: &err, debug: &debug);
204 QAudioDecoder::Error qerror = QAudioDecoder::ResourceError;
205 if (err->domain == GST_STREAM_ERROR) {
206 switch (err->code) {
207 case GST_STREAM_ERROR_DECRYPT:
208 case GST_STREAM_ERROR_DECRYPT_NOKEY:
209 qerror = QAudioDecoder::AccessDeniedError;
210 break;
211 case GST_STREAM_ERROR_FORMAT:
212 case GST_STREAM_ERROR_DEMUX:
213 case GST_STREAM_ERROR_DECODE:
214 case GST_STREAM_ERROR_WRONG_TYPE:
215 case GST_STREAM_ERROR_TYPE_NOT_FOUND:
216 case GST_STREAM_ERROR_CODEC_NOT_FOUND:
217 qerror = QAudioDecoder::FormatError;
218 break;
219 default:
220 break;
221 }
222 } else if (err->domain == GST_CORE_ERROR) {
223 switch (err->code) {
224 case GST_CORE_ERROR_MISSING_PLUGIN:
225 qerror = QAudioDecoder::FormatError;
226 break;
227 default:
228 break;
229 }
230 }
231
232 processInvalidMedia(errorCode: qerror, errorString: QString::fromUtf8(utf8: err->message));
233 g_error_free(error: err);
234 g_free(mem: debug);
235 }
236 }
237
238 return false;
239}
240
241QUrl QGstreamerAudioDecoder::source() const
242{
243 return mSource;
244}
245
246void QGstreamerAudioDecoder::setSource(const QUrl &fileName)
247{
248 stop();
249 mDevice = nullptr;
250 delete m_appSrc;
251 m_appSrc = nullptr;
252
253 bool isSignalRequired = (mSource != fileName);
254 mSource = fileName;
255 if (isSignalRequired)
256 emit sourceChanged();
257}
258
259QIODevice *QGstreamerAudioDecoder::sourceDevice() const
260{
261 return mDevice;
262}
263
264void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device)
265{
266 stop();
267 mSource.clear();
268 bool isSignalRequired = (mDevice != device);
269 mDevice = device;
270 if (isSignalRequired)
271 emit sourceChanged();
272}
273
274void QGstreamerAudioDecoder::start()
275{
276 addAppSink();
277
278 if (!mSource.isEmpty()) {
279 m_playbin.set(property: "uri", str: mSource.toEncoded().constData());
280 } else if (mDevice) {
281 // make sure we can read from device
282 if (!mDevice->isOpen() || !mDevice->isReadable()) {
283 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: QLatin1String("Unable to read from specified device"));
284 return;
285 }
286
287 if (!m_appSrc) {
288 auto maybeAppSrc = QGstAppSrc::create(parent: this);
289 if (maybeAppSrc) {
290 m_appSrc = maybeAppSrc.value();
291 } else {
292 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: maybeAppSrc.error());
293 return;
294 }
295 }
296
297 m_playbin.set(property: "uri", str: "appsrc://");
298 } else {
299 return;
300 }
301
302 // Set audio format
303 if (m_appSink) {
304 if (mFormat.isValid()) {
305 setAudioFlags(false);
306 auto caps = QGstUtils::capsForAudioFormat(format: mFormat);
307 gst_app_sink_set_caps(appsink: m_appSink, caps: caps.get());
308 } else {
309 // We want whatever the native audio format is
310 setAudioFlags(true);
311 gst_app_sink_set_caps(appsink: m_appSink, caps: nullptr);
312 }
313 }
314
315 if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
316 qWarning() << "GStreamer; Unable to start decoding process";
317 m_playbin.dumpGraph(fileName: "failed");
318 return;
319 }
320}
321
322void QGstreamerAudioDecoder::stop()
323{
324 m_playbin.setState(GST_STATE_NULL);
325 removeAppSink();
326
327 // GStreamer thread is stopped. Can safely access m_buffersAvailable
328 if (m_buffersAvailable != 0) {
329 m_buffersAvailable = 0;
330 emit bufferAvailableChanged(available: false);
331 }
332
333 if (m_position != -1) {
334 m_position = -1;
335 emit positionChanged(position: m_position);
336 }
337
338 if (m_duration != -1) {
339 m_duration = -1;
340 emit durationChanged(duration: m_duration);
341 }
342
343 setIsDecoding(false);
344}
345
346QAudioFormat QGstreamerAudioDecoder::audioFormat() const
347{
348 return mFormat;
349}
350
351void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format)
352{
353 if (mFormat != format) {
354 mFormat = format;
355 emit formatChanged(format: mFormat);
356 }
357}
358
359QAudioBuffer QGstreamerAudioDecoder::read()
360{
361 QAudioBuffer audioBuffer;
362
363 int buffersAvailable;
364 {
365 QMutexLocker locker(&m_buffersMutex);
366 buffersAvailable = m_buffersAvailable;
367
368 // need to decrement before pulling a buffer
369 // to make sure assert in QGstreamerAudioDecoderControl::new_buffer works
370 m_buffersAvailable--;
371 }
372
373
374 if (buffersAvailable) {
375 if (buffersAvailable == 1)
376 emit bufferAvailableChanged(available: false);
377
378 const char* bufferData = nullptr;
379 int bufferSize = 0;
380
381 GstSample *sample = gst_app_sink_pull_sample(appsink: m_appSink);
382 GstBuffer *buffer = gst_sample_get_buffer(sample);
383 GstMapInfo mapInfo;
384 gst_buffer_map(buffer, info: &mapInfo, flags: GST_MAP_READ);
385 bufferData = (const char*)mapInfo.data;
386 bufferSize = mapInfo.size;
387 QAudioFormat format = QGstUtils::audioFormatForSample(sample);
388
389 if (format.isValid()) {
390 // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer.
391 // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer.
392 qint64 position = getPositionFromBuffer(buffer);
393 audioBuffer = QAudioBuffer(QByteArray((const char*)bufferData, bufferSize), format, position);
394 position /= 1000; // convert to milliseconds
395 if (position != m_position) {
396 m_position = position;
397 emit positionChanged(position: m_position);
398 }
399 }
400 gst_buffer_unmap(buffer, info: &mapInfo);
401 gst_sample_unref(sample);
402 }
403
404 return audioBuffer;
405}
406
407bool QGstreamerAudioDecoder::bufferAvailable() const
408{
409 QMutexLocker locker(&m_buffersMutex);
410 return m_buffersAvailable > 0;
411}
412
413qint64 QGstreamerAudioDecoder::position() const
414{
415 return m_position;
416}
417
418qint64 QGstreamerAudioDecoder::duration() const
419{
420 return m_duration;
421}
422
423void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString)
424{
425 stop();
426 emit error(error: int(errorCode), errorString);
427}
428
429GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *, gpointer user_data)
430{
431 // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()."
432 QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder*>(user_data);
433
434 int buffersAvailable;
435 {
436 QMutexLocker locker(&decoder->m_buffersMutex);
437 buffersAvailable = decoder->m_buffersAvailable;
438 decoder->m_buffersAvailable++;
439 Q_ASSERT(decoder->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE);
440 }
441
442 if (!buffersAvailable)
443 decoder->bufferAvailableChanged(available: true);
444 decoder->bufferReady();
445 return GST_FLOW_OK;
446}
447
448void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio)
449{
450 int flags = m_playbin.getInt(property: "flags");
451 // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired
452 // it prevents audio format conversion
453 flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO);
454 flags |= GST_PLAY_FLAG_AUDIO;
455 if (wantNativeAudio)
456 flags |= GST_PLAY_FLAG_NATIVE_AUDIO;
457 m_playbin.set(property: "flags", i: flags);
458}
459
460void QGstreamerAudioDecoder::addAppSink()
461{
462 if (m_appSink)
463 return;
464
465 m_appSink = (GstAppSink*)gst_element_factory_make(factoryname: "appsink", name: nullptr);
466
467 GstAppSinkCallbacks callbacks;
468 memset(s: &callbacks, c: 0, n: sizeof(callbacks));
469 callbacks.new_sample = &new_sample;
470 gst_app_sink_set_callbacks(appsink: m_appSink, callbacks: &callbacks, user_data: this, notify: nullptr);
471 gst_app_sink_set_max_buffers(appsink: m_appSink, MAX_BUFFERS_IN_QUEUE);
472 gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE);
473
474 gst_bin_add(bin: m_outputBin.bin(), GST_ELEMENT(m_appSink));
475 gst_element_link(src: m_audioConvert.element(), GST_ELEMENT(m_appSink));
476}
477
478void QGstreamerAudioDecoder::removeAppSink()
479{
480 if (!m_appSink)
481 return;
482
483 gst_element_unlink(src: m_audioConvert.element(), GST_ELEMENT(m_appSink));
484 gst_bin_remove(bin: m_outputBin.bin(), GST_ELEMENT(m_appSink));
485
486 m_appSink = nullptr;
487}
488
489void QGstreamerAudioDecoder::updateDuration()
490{
491 int duration = m_playbin.duration() / 1000000;
492
493 if (m_duration != duration) {
494 m_duration = duration;
495 emit durationChanged(duration: m_duration);
496 }
497
498 if (m_duration > 0)
499 m_durationQueries = 0;
500
501 if (m_durationQueries > 0) {
502 //increase delay between duration requests
503 int delay = 25 << (5 - m_durationQueries);
504 QTimer::singleShot(msec: delay, receiver: this, SLOT(updateDuration()));
505 m_durationQueries--;
506 }
507}
508
509qint64 QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer* buffer)
510{
511 qint64 position = GST_BUFFER_TIMESTAMP(buffer);
512 if (position >= 0)
513 position = position / G_GINT64_CONSTANT(1000); // microseconds
514 else
515 position = -1;
516 return position;
517}
518
519QT_END_NAMESPACE
520
521#include "moc_qgstreameraudiodecoder_p.cpp"
522

source code of qtmultimedia/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp