1// Copyright (C) 2020 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3//#define DEBUG_DECODER
4
5#include <audio/qgstreameraudiodecoder_p.h>
6
7#include <common/qgst_debug_p.h>
8#include <common/qgstreamermessage_p.h>
9#include <common/qgstutils_p.h>
10#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
11
12#include <gst/gstvalue.h>
13#include <gst/base/gstbasesrc.h>
14
15#include <QtCore/qdatetime.h>
16#include <QtCore/qdebug.h>
17#include <QtCore/qsize.h>
18#include <QtCore/qtimer.h>
19#include <QtCore/qdebug.h>
20#include <QtCore/qdir.h>
21#include <QtCore/qstandardpaths.h>
22#include <QtCore/qurl.h>
23#include <QtCore/qloggingcategory.h>
24
25QT_BEGIN_NAMESPACE
26
27Q_STATIC_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder");
28
29typedef enum {
30 GST_PLAY_FLAG_VIDEO = 0x00000001,
31 GST_PLAY_FLAG_AUDIO = 0x00000002,
32 GST_PLAY_FLAG_TEXT = 0x00000004,
33 GST_PLAY_FLAG_VIS = 0x00000008,
34 GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010,
35 GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020,
36 GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040,
37 GST_PLAY_FLAG_DOWNLOAD = 0x00000080,
38 GST_PLAY_FLAG_BUFFERING = 0x000000100
39} GstPlayFlags;
40
41
42q23::expected<QPlatformAudioDecoder *, QString> QGstreamerAudioDecoder::create(QAudioDecoder *parent)
43{
44 static const auto error = qGstErrorMessageIfElementsNotAvailable(arg: "audioconvert", args: "playbin");
45 if (error)
46 return q23::unexpected{ *error };
47
48 return new QGstreamerAudioDecoder(parent);
49}
50
51QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent)
52 : QPlatformAudioDecoder(parent),
53 m_playbin{
54 QGstPipeline::createFromFactory(factory: "playbin3", name: "playbin"),
55 },
56 m_audioConvert{
57 QGstElement::createFromFactory(factory: "audioconvert", name: "audioconvert"),
58 }
59{
60 // Sort out messages
61 m_playbin.installMessageFilter(filter: this);
62
63 // Set the rest of the pipeline up
64 setAudioFlags(true);
65
66 m_outputBin = QGstBin::create(name: "audio-output-bin");
67 m_outputBin.add(ts: m_audioConvert);
68
69 // add ghostpad
70 m_outputBin.addGhostPad(child: m_audioConvert, name: "sink");
71
72 g_object_set(object: m_playbin.object(), first_property_name: "audio-sink", m_outputBin.element(), NULL);
73
74 // Set volume to 100%
75 gdouble volume = 1.0;
76 m_playbin.set(property: "volume", d: volume);
77}
78
79QGstreamerAudioDecoder::~QGstreamerAudioDecoder()
80{
81 stop();
82
83 m_playbin.removeMessageFilter(filter: this);
84}
85
86bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message)
87{
88 qCDebug(qLcGstreamerAudioDecoder) << "received bus message:" << message;
89
90 switch (message.type()) {
91 case GST_MESSAGE_DURATION:
92 return processBusMessageDuration(message);
93
94 case GST_MESSAGE_ERROR:
95 return processBusMessageError(message);
96
97 case GST_MESSAGE_WARNING:
98 return processBusMessageWarning(message);
99
100 case GST_MESSAGE_INFO:
101 return processBusMessageInfo(message);
102
103 case GST_MESSAGE_EOS:
104 return processBusMessageEOS(message);
105
106 case GST_MESSAGE_STATE_CHANGED:
107 return processBusMessageStateChanged(message);
108
109 case GST_MESSAGE_STREAMS_SELECTED:
110 return processBusMessageStreamsSelected(message);
111
112 default:
113 return false;
114 }
115}
116
117bool QGstreamerAudioDecoder::canReadQrc() const
118{
119 return true;
120}
121
122bool QGstreamerAudioDecoder::processBusMessageError(const QGstreamerMessage &message)
123{
124 qCDebug(qLcGstreamerAudioDecoder) << " error" << QCompactGstMessageAdaptor(message);
125
126 QUniqueGErrorHandle err;
127 QGString debug;
128 gst_message_parse_error(message: message.message(), gerror: &err, debug: &debug);
129
130 if (message.source() == m_playbin) {
131 if (err.get()->domain == GST_STREAM_ERROR
132 && err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND)
133 processInvalidMedia(errorCode: QAudioDecoder::FormatError,
134 errorString: tr(s: "Cannot play stream of type: <unknown>"));
135 else
136 processInvalidMedia(errorCode: QAudioDecoder::ResourceError,
137 errorString: QString::fromUtf8(utf8: err.get()->message));
138 } else {
139 QAudioDecoder::Error qerror = QAudioDecoder::ResourceError;
140 if (err.get()->domain == GST_STREAM_ERROR) {
141 switch (err.get()->code) {
142 case GST_STREAM_ERROR_DECRYPT:
143 case GST_STREAM_ERROR_DECRYPT_NOKEY:
144 qerror = QAudioDecoder::AccessDeniedError;
145 break;
146 case GST_STREAM_ERROR_FORMAT:
147 case GST_STREAM_ERROR_DEMUX:
148 case GST_STREAM_ERROR_DECODE:
149 case GST_STREAM_ERROR_WRONG_TYPE:
150 case GST_STREAM_ERROR_TYPE_NOT_FOUND:
151 case GST_STREAM_ERROR_CODEC_NOT_FOUND:
152 qerror = QAudioDecoder::FormatError;
153 break;
154 default:
155 break;
156 }
157 } else if (err.get()->domain == GST_CORE_ERROR) {
158 switch (err.get()->code) {
159 case GST_CORE_ERROR_MISSING_PLUGIN:
160 qerror = QAudioDecoder::FormatError;
161 break;
162 default:
163 break;
164 }
165 }
166
167 processInvalidMedia(errorCode: qerror, errorString: QString::fromUtf8(utf8: err.get()->message));
168 }
169
170 return false;
171}
172
173bool QGstreamerAudioDecoder::processBusMessageDuration(const QGstreamerMessage &)
174{
175 updateDuration();
176 return false;
177}
178
179bool QGstreamerAudioDecoder::processBusMessageWarning(const QGstreamerMessage &message)
180{
181 qCWarning(qLcGstreamerAudioDecoder) << "Warning:" << QCompactGstMessageAdaptor(message);
182 return false;
183}
184
185bool QGstreamerAudioDecoder::processBusMessageInfo(const QGstreamerMessage &message)
186{
187 if (qLcGstreamerAudioDecoder().isDebugEnabled())
188 qCWarning(qLcGstreamerAudioDecoder) << "Info:" << QCompactGstMessageAdaptor(message);
189 return false;
190}
191
192bool QGstreamerAudioDecoder::processBusMessageEOS(const QGstreamerMessage &)
193{
194 m_playbin.setState(GST_STATE_NULL);
195 finished();
196 return false;
197}
198
199bool QGstreamerAudioDecoder::processBusMessageStateChanged(const QGstreamerMessage &message)
200{
201 if (message.source() != m_playbin)
202 return false;
203
204 GstState oldState;
205 GstState newState;
206 GstState pending;
207
208 gst_message_parse_state_changed(message: message.message(), oldstate: &oldState, newstate: &newState, pending: &pending);
209
210 bool isDecoding = false;
211 switch (newState) {
212 case GST_STATE_VOID_PENDING:
213 case GST_STATE_NULL:
214 case GST_STATE_READY:
215 break;
216 case GST_STATE_PLAYING:
217 isDecoding = true;
218 break;
219 case GST_STATE_PAUSED:
220 isDecoding = true;
221
222 // gstreamer doesn't give a reliable indication the duration
223 // information is ready, GST_MESSAGE_DURATION is not sent by most elements
224 // the duration is queried up to 5 times with increasing delay
225 m_durationQueries = 5;
226 updateDuration();
227 break;
228 }
229
230 setIsDecoding(isDecoding);
231 return false;
232}
233
234bool QGstreamerAudioDecoder::processBusMessageStreamsSelected(const QGstreamerMessage &message)
235{
236 using namespace Qt::StringLiterals;
237
238 QGstStreamCollectionHandle collection;
239 gst_message_parse_streams_selected(message: const_cast<GstMessage *>(message.message()), collection: &collection);
240
241 bool hasAudio = false;
242 qForeachStreamInCollection(collection, f: [&](GstStream *stream) {
243 GstStreamType type = gst_stream_get_stream_type(stream);
244 if (type == GstStreamType::GST_STREAM_TYPE_AUDIO)
245 hasAudio = true;
246 });
247
248 if (!hasAudio)
249 processInvalidMedia(errorCode: QAudioDecoder::FormatError, errorString: u"No audio track in media"_s);
250
251 return false;
252}
253
254QUrl QGstreamerAudioDecoder::source() const
255{
256 return mSource;
257}
258
259void QGstreamerAudioDecoder::setSource(const QUrl &fileName)
260{
261 stop();
262 mDevice = nullptr;
263
264 bool isSignalRequired = (mSource != fileName);
265 mSource = fileName;
266 if (isSignalRequired)
267 sourceChanged();
268}
269
270QIODevice *QGstreamerAudioDecoder::sourceDevice() const
271{
272 return mDevice;
273}
274
275void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device)
276{
277 stop();
278 mSource.clear();
279 bool isSignalRequired = (mDevice != device);
280 mDevice = device;
281 if (isSignalRequired)
282 sourceChanged();
283}
284
285void QGstreamerAudioDecoder::start()
286{
287 addAppSink();
288
289 if (!mSource.isEmpty()) {
290 m_playbin.set(property: "uri", str: mSource.toEncoded().constData());
291 } else if (mDevice) {
292 // make sure we can read from device
293 if (!mDevice->isOpen() || !mDevice->isReadable()) {
294 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: QLatin1String("Unable to read from specified device"));
295 return;
296 }
297
298 QUrl streamURL = qGstRegisterQIODevice(mDevice);
299 m_playbin.set(property: "uri", str: streamURL.toEncoded().constData());
300 } else {
301 return;
302 }
303
304 // Set audio format
305 if (m_appSink) {
306 if (mFormat.isValid()) {
307 setAudioFlags(false);
308 auto caps = QGstUtils::capsForAudioFormat(format: mFormat);
309 m_appSink.setCaps(caps);
310 } else {
311 // We want whatever the native audio format is
312 setAudioFlags(true);
313 m_appSink.setCaps({});
314 }
315 }
316
317 if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
318 qWarning() << "GStreamer; Unable to start decoding process";
319 m_playbin.dumpGraph(fileNamePrefix: "failed");
320 return;
321 }
322}
323
324void QGstreamerAudioDecoder::stop()
325{
326 m_playbin.setState(GST_STATE_NULL);
327 m_currentSessionId += 1;
328 removeAppSink();
329
330 // GStreamer thread is stopped. Can safely access m_buffersAvailable
331 if (m_buffersAvailable != 0) {
332 m_buffersAvailable = 0;
333 bufferAvailableChanged(available: false);
334 }
335
336 if (m_position != invalidPosition) {
337 m_position = invalidPosition;
338 positionChanged(position: m_position.count());
339 }
340
341 if (m_duration != invalidDuration) {
342 m_duration = invalidDuration;
343 durationChanged(duration: m_duration.count());
344 }
345
346 setIsDecoding(false);
347}
348
349QAudioFormat QGstreamerAudioDecoder::audioFormat() const
350{
351 return mFormat;
352}
353
354void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format)
355{
356 if (mFormat != format) {
357 mFormat = format;
358 formatChanged(format: mFormat);
359 }
360}
361
362QAudioBuffer QGstreamerAudioDecoder::read()
363{
364 using namespace std::chrono;
365
366 QAudioBuffer audioBuffer;
367
368 if (m_buffersAvailable == 0)
369 return audioBuffer;
370
371 m_buffersAvailable -= 1;
372
373 if (m_buffersAvailable == 0)
374 bufferAvailableChanged(available: false);
375
376 QGstSampleHandle sample = m_appSink.pullSample();
377 GstBuffer *buffer = gst_sample_get_buffer(sample: sample.get());
378 GstMapInfo mapInfo;
379 gst_buffer_map(buffer, info: &mapInfo, flags: GST_MAP_READ);
380 const char *bufferData = (const char *)mapInfo.data;
381 int bufferSize = mapInfo.size;
382 QAudioFormat format = QGstUtils::audioFormatForSample(sample: sample.get());
383
384 if (format.isValid()) {
385 // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer.
386 // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer.
387 nanoseconds position = getPositionFromBuffer(buffer);
388 audioBuffer = QAudioBuffer{
389 QByteArray(bufferData, bufferSize),
390 format,
391 round<microseconds>(d: position).count(),
392 };
393 milliseconds positionInMs = round<milliseconds>(d: position);
394 if (position != m_position) {
395 m_position = positionInMs;
396 positionChanged(position: m_position.count());
397 }
398 }
399 gst_buffer_unmap(buffer, info: &mapInfo);
400
401 return audioBuffer;
402}
403
404qint64 QGstreamerAudioDecoder::position() const
405{
406 return m_position.count();
407}
408
409qint64 QGstreamerAudioDecoder::duration() const
410{
411 return m_duration.count();
412}
413
414void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString)
415{
416 stop();
417 error(error: int(errorCode), errorString);
418}
419
420GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *)
421{
422 // "Note that the preroll buffer will also be returned as the first buffer when calling
423 // gst_app_sink_pull_buffer()."
424
425 QMetaObject::invokeMethod(object: this, function: [this, sessionId = m_currentSessionId] {
426 if (sessionId != m_currentSessionId)
427 return; // stop()ed before message is executed
428
429 m_buffersAvailable += 1;
430 bufferAvailableChanged(available: true);
431 bufferReady();
432 });
433
434 return GST_FLOW_OK;
435}
436
437GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data)
438{
439 QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data);
440 qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample";
441 return decoder->newSample(sink);
442}
443
444void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio)
445{
446 int flags = m_playbin.getInt(property: "flags");
447 // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired
448 // it prevents audio format conversion
449 flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO);
450 flags |= GST_PLAY_FLAG_AUDIO;
451 if (wantNativeAudio)
452 flags |= GST_PLAY_FLAG_NATIVE_AUDIO;
453 m_playbin.set(property: "flags", i: flags);
454}
455
456void QGstreamerAudioDecoder::addAppSink()
457{
458 using namespace std::chrono_literals;
459
460 if (m_appSink)
461 return;
462
463 qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink";
464 m_appSink = QGstAppSink::create(name: "decoderAppSink");
465 GstAppSinkCallbacks callbacks{};
466 callbacks.new_sample = new_sample;
467 m_appSink.setCallbacks(callbacks, user_data: this, notify: nullptr);
468
469#if GST_CHECK_VERSION(1, 24, 0)
470 static constexpr auto maxBufferTime = 500ms;
471 m_appSink.setMaxBufferTime(maxBufferTime);
472#else
473 static constexpr int maxBuffers = 16;
474 m_appSink.setMaxBuffers(maxBuffers);
475#endif
476
477 static constexpr bool sync = false;
478 m_appSink.setSync(sync);
479
480 m_audioConvert.src().modifyPipelineInIdleProbe(f: [&] {
481 m_outputBin.add(ts: m_appSink);
482 qLinkGstElements(ts: m_audioConvert, ts: m_appSink);
483 });
484}
485
486void QGstreamerAudioDecoder::removeAppSink()
487{
488 if (!m_appSink)
489 return;
490
491 qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink";
492
493 m_audioConvert.src().modifyPipelineInIdleProbe(f: [&] {
494 qUnlinkGstElements(ts: m_audioConvert, ts: m_appSink);
495 m_outputBin.stopAndRemoveElements(ts&: m_appSink);
496 });
497
498 m_appSink = {};
499}
500
501void QGstreamerAudioDecoder::updateDuration()
502{
503 std::optional<std::chrono::milliseconds> duration = m_playbin.durationInMs();
504 if (!duration)
505 duration = invalidDuration;
506
507 if (m_duration != duration) {
508 m_duration = *duration;
509 durationChanged(duration: m_duration.count());
510 }
511
512 if (m_duration.count() > 0)
513 m_durationQueries = 0;
514
515 if (m_durationQueries > 0) {
516 //increase delay between duration requests
517 int delay = 25 << (5 - m_durationQueries);
518 QTimer::singleShot(interval: delay, receiver: this, slot: &QGstreamerAudioDecoder::updateDuration);
519 m_durationQueries--;
520 }
521}
522
523std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer)
524{
525 using namespace std::chrono;
526 using namespace std::chrono_literals;
527 nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) };
528 if (position >= 0ns)
529 return position;
530 else
531 return invalidPosition;
532}
533
534QT_END_NAMESPACE
535
536#include "moc_qgstreameraudiodecoder_p.cpp"
537

source code of qtmultimedia/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp