1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39//#define DEBUG_DECODER
40
41#include "qgstreameraudiodecodersession.h"
42#include <private/qgstreamerbushelper_p.h>
43
44#include <private/qgstutils_p.h>
45
46#include <gst/gstvalue.h>
47#include <gst/base/gstbasesrc.h>
48
49#include <QtCore/qdatetime.h>
50#include <QtCore/qdebug.h>
51#include <QtCore/qsize.h>
52#include <QtCore/qtimer.h>
53#include <QtCore/qdebug.h>
54#include <QtCore/qdir.h>
55#include <QtCore/qstandardpaths.h>
56#include <QtCore/qurl.h>
57
58#define MAX_BUFFERS_IN_QUEUE 4
59
60QT_BEGIN_NAMESPACE
61
62typedef enum {
63 GST_PLAY_FLAG_VIDEO = 0x00000001,
64 GST_PLAY_FLAG_AUDIO = 0x00000002,
65 GST_PLAY_FLAG_TEXT = 0x00000004,
66 GST_PLAY_FLAG_VIS = 0x00000008,
67 GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010,
68 GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020,
69 GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040,
70 GST_PLAY_FLAG_DOWNLOAD = 0x00000080,
71 GST_PLAY_FLAG_BUFFERING = 0x000000100
72} GstPlayFlags;
73
74QGstreamerAudioDecoderSession::QGstreamerAudioDecoderSession(QObject *parent)
75 : QObject(parent),
76 m_state(QAudioDecoder::StoppedState),
77 m_pendingState(QAudioDecoder::StoppedState),
78 m_busHelper(0),
79 m_bus(0),
80 m_playbin(0),
81 m_outputBin(0),
82 m_audioConvert(0),
83 m_appSink(0),
84#if QT_CONFIG(gstreamer_app)
85 m_appSrc(0),
86#endif
87 mDevice(0),
88 m_buffersAvailable(0),
89 m_position(-1),
90 m_duration(-1),
91 m_durationQueries(0)
92{
93 // Create pipeline here
94 m_playbin = gst_element_factory_make(QT_GSTREAMER_PLAYBIN_ELEMENT_NAME, NULL);
95
96 if (m_playbin != 0) {
97 // Sort out messages
98 m_bus = gst_element_get_bus(element: m_playbin);
99 m_busHelper = new QGstreamerBusHelper(m_bus, this);
100 m_busHelper->installMessageFilter(filter: this);
101
102 // Set the rest of the pipeline up
103 setAudioFlags(true);
104
105 m_audioConvert = gst_element_factory_make(factoryname: "audioconvert", NULL);
106
107 m_outputBin = gst_bin_new(name: "audio-output-bin");
108 gst_bin_add(GST_BIN(m_outputBin), element: m_audioConvert);
109
110 // add ghostpad
111 GstPad *pad = gst_element_get_static_pad(element: m_audioConvert, name: "sink");
112 Q_ASSERT(pad);
113 gst_element_add_pad(GST_ELEMENT(m_outputBin), pad: gst_ghost_pad_new(name: "sink", target: pad));
114 gst_object_unref(GST_OBJECT(pad));
115
116 g_object_set(G_OBJECT(m_playbin), first_property_name: "audio-sink", m_outputBin, NULL);
117#if QT_CONFIG(gstreamer_app)
118 g_signal_connect(G_OBJECT(m_playbin), "deep-notify::source", (GCallback) &QGstreamerAudioDecoderSession::configureAppSrcElement, (gpointer)this);
119#endif
120
121 // Set volume to 100%
122 gdouble volume = 1.0;
123 g_object_set(G_OBJECT(m_playbin), first_property_name: "volume", volume, NULL);
124 }
125}
126
127QGstreamerAudioDecoderSession::~QGstreamerAudioDecoderSession()
128{
129 if (m_playbin) {
130 stop();
131
132 delete m_busHelper;
133#if QT_CONFIG(gstreamer_app)
134 delete m_appSrc;
135#endif
136 gst_object_unref(GST_OBJECT(m_bus));
137 gst_object_unref(GST_OBJECT(m_playbin));
138 }
139}
140
141#if QT_CONFIG(gstreamer_app)
142void QGstreamerAudioDecoderSession::configureAppSrcElement(GObject* object, GObject *orig, GParamSpec *pspec, QGstreamerAudioDecoderSession* self)
143{
144 Q_UNUSED(object);
145 Q_UNUSED(pspec);
146
147 // In case we switch from appsrc to not
148 if (!self->appsrc())
149 return;
150
151 GstElement *appsrc;
152 g_object_get(object: orig, first_property_name: "source", &appsrc, NULL);
153
154 if (!self->appsrc()->setup(appsrc))
155 qWarning()<<"Could not setup appsrc element";
156
157 g_object_unref(G_OBJECT(appsrc));
158}
159#endif
160
161bool QGstreamerAudioDecoderSession::processBusMessage(const QGstreamerMessage &message)
162{
163 GstMessage* gm = message.rawMessage();
164 if (gm) {
165 if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_DURATION) {
166 updateDuration();
167 } else if (GST_MESSAGE_SRC(gm) == GST_OBJECT_CAST(m_playbin)) {
168 switch (GST_MESSAGE_TYPE(gm)) {
169 case GST_MESSAGE_STATE_CHANGED:
170 {
171 GstState oldState;
172 GstState newState;
173 GstState pending;
174
175 gst_message_parse_state_changed(message: gm, oldstate: &oldState, newstate: &newState, pending: &pending);
176
177#ifdef DEBUG_DECODER
178 QStringList states;
179 states << "GST_STATE_VOID_PENDING" << "GST_STATE_NULL" << "GST_STATE_READY" << "GST_STATE_PAUSED" << "GST_STATE_PLAYING";
180
181 qDebug() << QString("state changed: old: %1 new: %2 pending: %3") \
182 .arg(states[oldState]) \
183 .arg(states[newState]) \
184 .arg(states[pending]) << "internal" << m_state;
185#endif
186
187 QAudioDecoder::State prevState = m_state;
188
189 switch (newState) {
190 case GST_STATE_VOID_PENDING:
191 case GST_STATE_NULL:
192 m_state = QAudioDecoder::StoppedState;
193 break;
194 case GST_STATE_READY:
195 m_state = QAudioDecoder::StoppedState;
196 break;
197 case GST_STATE_PLAYING:
198 m_state = QAudioDecoder::DecodingState;
199 break;
200 case GST_STATE_PAUSED:
201 m_state = QAudioDecoder::DecodingState;
202
203 //gstreamer doesn't give a reliable indication the duration
204 //information is ready, GST_MESSAGE_DURATION is not sent by most elements
205 //the duration is queried up to 5 times with increasing delay
206 m_durationQueries = 5;
207 updateDuration();
208 break;
209 }
210
211 if (prevState != m_state)
212 emit stateChanged(newState: m_state);
213 }
214 break;
215
216 case GST_MESSAGE_EOS:
217 m_pendingState = m_state = QAudioDecoder::StoppedState;
218 emit finished();
219 emit stateChanged(newState: m_state);
220 break;
221
222 case GST_MESSAGE_ERROR: {
223 GError *err;
224 gchar *debug;
225 gst_message_parse_error(message: gm, gerror: &err, debug: &debug);
226 if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND)
227 processInvalidMedia(errorCode: QAudioDecoder::FormatError, errorString: tr(s: "Cannot play stream of type: <unknown>"));
228 else
229 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: QString::fromUtf8(str: err->message));
230 qWarning() << "Error:" << QString::fromUtf8(str: err->message);
231 g_error_free(error: err);
232 g_free(mem: debug);
233 }
234 break;
235 case GST_MESSAGE_WARNING:
236 {
237 GError *err;
238 gchar *debug;
239 gst_message_parse_warning (message: gm, gerror: &err, debug: &debug);
240 qWarning() << "Warning:" << QString::fromUtf8(str: err->message);
241 g_error_free (error: err);
242 g_free (mem: debug);
243 }
244 break;
245#ifdef DEBUG_DECODER
246 case GST_MESSAGE_INFO:
247 {
248 GError *err;
249 gchar *debug;
250 gst_message_parse_info (gm, &err, &debug);
251 qDebug() << "Info:" << QString::fromUtf8(err->message);
252 g_error_free (err);
253 g_free (debug);
254 }
255 break;
256#endif
257 default:
258 break;
259 }
260 } else if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_ERROR) {
261 GError *err;
262 gchar *debug;
263 gst_message_parse_error(message: gm, gerror: &err, debug: &debug);
264 QAudioDecoder::Error qerror = QAudioDecoder::ResourceError;
265 if (err->domain == GST_STREAM_ERROR) {
266 switch (err->code) {
267 case GST_STREAM_ERROR_DECRYPT:
268 case GST_STREAM_ERROR_DECRYPT_NOKEY:
269 qerror = QAudioDecoder::AccessDeniedError;
270 break;
271 case GST_STREAM_ERROR_FORMAT:
272 case GST_STREAM_ERROR_DEMUX:
273 case GST_STREAM_ERROR_DECODE:
274 case GST_STREAM_ERROR_WRONG_TYPE:
275 case GST_STREAM_ERROR_TYPE_NOT_FOUND:
276 case GST_STREAM_ERROR_CODEC_NOT_FOUND:
277 qerror = QAudioDecoder::FormatError;
278 break;
279 default:
280 break;
281 }
282 } else if (err->domain == GST_CORE_ERROR) {
283 switch (err->code) {
284 case GST_CORE_ERROR_MISSING_PLUGIN:
285 qerror = QAudioDecoder::FormatError;
286 break;
287 default:
288 break;
289 }
290 }
291
292 processInvalidMedia(errorCode: qerror, errorString: QString::fromUtf8(str: err->message));
293 g_error_free(error: err);
294 g_free(mem: debug);
295 }
296 }
297
298 return false;
299}
300
301QString QGstreamerAudioDecoderSession::sourceFilename() const
302{
303 return mSource;
304}
305
306void QGstreamerAudioDecoderSession::setSourceFilename(const QString &fileName)
307{
308 stop();
309 mDevice = 0;
310#if QT_CONFIG(gstreamer_app)
311 if (m_appSrc)
312 m_appSrc->deleteLater();
313 m_appSrc = 0;
314#endif
315
316 bool isSignalRequired = (mSource != fileName);
317 mSource = fileName;
318 if (isSignalRequired)
319 emit sourceChanged();
320}
321
322QIODevice *QGstreamerAudioDecoderSession::sourceDevice() const
323{
324 return mDevice;
325}
326
327void QGstreamerAudioDecoderSession::setSourceDevice(QIODevice *device)
328{
329 stop();
330 mSource.clear();
331 bool isSignalRequired = (mDevice != device);
332 mDevice = device;
333 if (isSignalRequired)
334 emit sourceChanged();
335}
336
337void QGstreamerAudioDecoderSession::start()
338{
339 if (!m_playbin) {
340 processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: "Playbin element is not valid");
341 return;
342 }
343
344 addAppSink();
345
346 if (!mSource.isEmpty()) {
347 g_object_set(G_OBJECT(m_playbin), first_property_name: "uri", QUrl::fromLocalFile(localfile: mSource).toEncoded().constData(), NULL);
348 } else if (mDevice) {
349#if QT_CONFIG(gstreamer_app)
350 // make sure we can read from device
351 if (!mDevice->isOpen() || !mDevice->isReadable()) {
352 processInvalidMedia(errorCode: QAudioDecoder::AccessDeniedError, errorString: "Unable to read from specified device");
353 return;
354 }
355
356 if (!m_appSrc)
357 m_appSrc = new QGstAppSrc(this);
358 m_appSrc->setStream(mDevice);
359
360 g_object_set(G_OBJECT(m_playbin), first_property_name: "uri", "appsrc://", NULL);
361#endif
362 } else {
363 return;
364 }
365
366 // Set audio format
367 if (m_appSink) {
368 if (mFormat.isValid()) {
369 setAudioFlags(false);
370 GstCaps *caps = QGstUtils::capsForAudioFormat(format: mFormat);
371 gst_app_sink_set_caps(appsink: m_appSink, caps);
372 gst_caps_unref(caps);
373 } else {
374 // We want whatever the native audio format is
375 setAudioFlags(true);
376 gst_app_sink_set_caps(appsink: m_appSink, NULL);
377 }
378 }
379
380 m_pendingState = QAudioDecoder::DecodingState;
381 if (gst_element_set_state(element: m_playbin, state: GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
382 qWarning() << "GStreamer; Unable to start decoding process";
383 m_pendingState = m_state = QAudioDecoder::StoppedState;
384
385 emit stateChanged(newState: m_state);
386 }
387}
388
389void QGstreamerAudioDecoderSession::stop()
390{
391 if (m_playbin) {
392 gst_element_set_state(element: m_playbin, state: GST_STATE_NULL);
393 removeAppSink();
394
395 QAudioDecoder::State oldState = m_state;
396 m_pendingState = m_state = QAudioDecoder::StoppedState;
397
398 // GStreamer thread is stopped. Can safely access m_buffersAvailable
399 if (m_buffersAvailable != 0) {
400 m_buffersAvailable = 0;
401 emit bufferAvailableChanged(available: false);
402 }
403
404 if (m_position != -1) {
405 m_position = -1;
406 emit positionChanged(position: m_position);
407 }
408
409 if (m_duration != -1) {
410 m_duration = -1;
411 emit durationChanged(duration: m_duration);
412 }
413
414 if (oldState != m_state)
415 emit stateChanged(newState: m_state);
416 }
417}
418
419QAudioFormat QGstreamerAudioDecoderSession::audioFormat() const
420{
421 return mFormat;
422}
423
424void QGstreamerAudioDecoderSession::setAudioFormat(const QAudioFormat &format)
425{
426 if (mFormat != format) {
427 mFormat = format;
428 emit formatChanged(format: mFormat);
429 }
430}
431
432QAudioBuffer QGstreamerAudioDecoderSession::read()
433{
434 QAudioBuffer audioBuffer;
435
436 int buffersAvailable;
437 {
438 QMutexLocker locker(&m_buffersMutex);
439 buffersAvailable = m_buffersAvailable;
440
441 // need to decrement before pulling a buffer
442 // to make sure assert in QGstreamerAudioDecoderSession::new_buffer works
443 m_buffersAvailable--;
444 }
445
446
447 if (buffersAvailable) {
448 if (buffersAvailable == 1)
449 emit bufferAvailableChanged(available: false);
450
451 const char* bufferData = 0;
452 int bufferSize = 0;
453
454#if GST_CHECK_VERSION(1,0,0)
455 GstSample *sample = gst_app_sink_pull_sample(appsink: m_appSink);
456 GstBuffer *buffer = gst_sample_get_buffer(sample);
457 GstMapInfo mapInfo;
458 gst_buffer_map(buffer, info: &mapInfo, flags: GST_MAP_READ);
459 bufferData = (const char*)mapInfo.data;
460 bufferSize = mapInfo.size;
461 QAudioFormat format = QGstUtils::audioFormatForSample(sample);
462#else
463 GstBuffer *buffer = gst_app_sink_pull_buffer(m_appSink);
464 bufferData = (const char*)buffer->data;
465 bufferSize = buffer->size;
466 QAudioFormat format = QGstUtils::audioFormatForBuffer(buffer);
467#endif
468
469 if (format.isValid()) {
470 // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer.
471 // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer.
472 qint64 position = getPositionFromBuffer(buffer);
473 audioBuffer = QAudioBuffer(QByteArray((const char*)bufferData, bufferSize), format, position);
474 position /= 1000; // convert to milliseconds
475 if (position != m_position) {
476 m_position = position;
477 emit positionChanged(position: m_position);
478 }
479 }
480#if GST_CHECK_VERSION(1,0,0)
481 gst_buffer_unmap(buffer, info: &mapInfo);
482 gst_sample_unref(sample);
483#else
484 gst_buffer_unref(buffer);
485#endif
486 }
487
488 return audioBuffer;
489}
490
491bool QGstreamerAudioDecoderSession::bufferAvailable() const
492{
493 QMutexLocker locker(&m_buffersMutex);
494 return m_buffersAvailable > 0;
495}
496
497qint64 QGstreamerAudioDecoderSession::position() const
498{
499 return m_position;
500}
501
502qint64 QGstreamerAudioDecoderSession::duration() const
503{
504 return m_duration;
505}
506
507void QGstreamerAudioDecoderSession::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString)
508{
509 stop();
510 emit error(error: int(errorCode), errorString);
511}
512
513GstFlowReturn QGstreamerAudioDecoderSession::new_sample(GstAppSink *, gpointer user_data)
514{
515 // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()."
516 QGstreamerAudioDecoderSession *session = reinterpret_cast<QGstreamerAudioDecoderSession*>(user_data);
517
518 int buffersAvailable;
519 {
520 QMutexLocker locker(&session->m_buffersMutex);
521 buffersAvailable = session->m_buffersAvailable;
522 session->m_buffersAvailable++;
523 Q_ASSERT(session->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE);
524 }
525
526 if (!buffersAvailable)
527 QMetaObject::invokeMethod(obj: session, member: "bufferAvailableChanged", type: Qt::QueuedConnection, Q_ARG(bool, true));
528 QMetaObject::invokeMethod(obj: session, member: "bufferReady", type: Qt::QueuedConnection);
529 return GST_FLOW_OK;
530}
531
532void QGstreamerAudioDecoderSession::setAudioFlags(bool wantNativeAudio)
533{
534 int flags = 0;
535 if (m_playbin) {
536 g_object_get(G_OBJECT(m_playbin), first_property_name: "flags", &flags, NULL);
537 // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired
538 // it prevents audio format conversion
539 flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO);
540 flags |= GST_PLAY_FLAG_AUDIO;
541 if (wantNativeAudio)
542 flags |= GST_PLAY_FLAG_NATIVE_AUDIO;
543 g_object_set(G_OBJECT(m_playbin), first_property_name: "flags", flags, NULL);
544 }
545}
546
547void QGstreamerAudioDecoderSession::addAppSink()
548{
549 if (m_appSink)
550 return;
551
552 m_appSink = (GstAppSink*)gst_element_factory_make(factoryname: "appsink", NULL);
553
554 GstAppSinkCallbacks callbacks;
555 memset(s: &callbacks, c: 0, n: sizeof(callbacks));
556#if GST_CHECK_VERSION(1,0,0)
557 callbacks.new_sample = &new_sample;
558#else
559 callbacks.new_buffer = &new_sample;
560#endif
561 gst_app_sink_set_callbacks(appsink: m_appSink, callbacks: &callbacks, user_data: this, NULL);
562 gst_app_sink_set_max_buffers(appsink: m_appSink, MAX_BUFFERS_IN_QUEUE);
563 gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE);
564
565 gst_bin_add(GST_BIN(m_outputBin), GST_ELEMENT(m_appSink));
566 gst_element_link(src: m_audioConvert, GST_ELEMENT(m_appSink));
567}
568
569void QGstreamerAudioDecoderSession::removeAppSink()
570{
571 if (!m_appSink)
572 return;
573
574 gst_element_unlink(src: m_audioConvert, GST_ELEMENT(m_appSink));
575 gst_bin_remove(GST_BIN(m_outputBin), GST_ELEMENT(m_appSink));
576
577 m_appSink = 0;
578}
579
580void QGstreamerAudioDecoderSession::updateDuration()
581{
582 gint64 gstDuration = 0;
583 int duration = -1;
584
585 if (m_playbin && qt_gst_element_query_duration(element: m_playbin, format: GST_FORMAT_TIME, cur: &gstDuration))
586 duration = gstDuration / 1000000;
587
588 if (m_duration != duration) {
589 m_duration = duration;
590 emit durationChanged(duration: m_duration);
591 }
592
593 if (m_duration > 0)
594 m_durationQueries = 0;
595
596 if (m_durationQueries > 0) {
597 //increase delay between duration requests
598 int delay = 25 << (5 - m_durationQueries);
599 QTimer::singleShot(msec: delay, receiver: this, SLOT(updateDuration()));
600 m_durationQueries--;
601 }
602}
603
604qint64 QGstreamerAudioDecoderSession::getPositionFromBuffer(GstBuffer* buffer)
605{
606 qint64 position = GST_BUFFER_TIMESTAMP(buffer);
607 if (position >= 0)
608 position = position / G_GINT64_CONSTANT(1000); // microseconds
609 else
610 position = -1;
611 return position;
612}
613
614QT_END_NAMESPACE
615

source code of qtmultimedia/src/plugins/gstreamer/audiodecoder/qgstreameraudiodecodersession.cpp