1 | // Copyright (C) 2020 The Qt Company Ltd. |
---|---|
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | //#define DEBUG_DECODER |
4 | |
5 | #include <audio/qgstreameraudiodecoder_p.h> |
6 | |
7 | #include <common/qgst_debug_p.h> |
8 | #include <common/qgstreamermessage_p.h> |
9 | #include <common/qgstutils_p.h> |
10 | #include <uri_handler/qgstreamer_qiodevice_handler_p.h> |
11 | |
12 | #include <gst/gstvalue.h> |
13 | #include <gst/base/gstbasesrc.h> |
14 | |
15 | #include <QtCore/qdatetime.h> |
16 | #include <QtCore/qdebug.h> |
17 | #include <QtCore/qsize.h> |
18 | #include <QtCore/qtimer.h> |
19 | #include <QtCore/qdebug.h> |
20 | #include <QtCore/qdir.h> |
21 | #include <QtCore/qstandardpaths.h> |
22 | #include <QtCore/qurl.h> |
23 | #include <QtCore/qloggingcategory.h> |
24 | |
25 | QT_BEGIN_NAMESPACE |
26 | |
27 | static Q_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder"); |
28 | |
29 | typedef enum { |
30 | GST_PLAY_FLAG_VIDEO = 0x00000001, |
31 | GST_PLAY_FLAG_AUDIO = 0x00000002, |
32 | GST_PLAY_FLAG_TEXT = 0x00000004, |
33 | GST_PLAY_FLAG_VIS = 0x00000008, |
34 | GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010, |
35 | GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020, |
36 | GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040, |
37 | GST_PLAY_FLAG_DOWNLOAD = 0x00000080, |
38 | GST_PLAY_FLAG_BUFFERING = 0x000000100 |
39 | } GstPlayFlags; |
40 | |
41 | |
42 | QMaybe<QPlatformAudioDecoder *> QGstreamerAudioDecoder::create(QAudioDecoder *parent) |
43 | { |
44 | static const auto error = qGstErrorMessageIfElementsNotAvailable(arg: "audioconvert", args: "playbin"); |
45 | if (error) |
46 | return *error; |
47 | |
48 | return new QGstreamerAudioDecoder(parent); |
49 | } |
50 | |
51 | QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) |
52 | : QPlatformAudioDecoder(parent), |
53 | m_playbin{ |
54 | QGstPipeline::createFromFactory(factory: "playbin3", name: "playbin"), |
55 | }, |
56 | m_audioConvert{ |
57 | QGstElement::createFromFactory(factory: "audioconvert", name: "audioconvert"), |
58 | } |
59 | { |
60 | // Sort out messages |
61 | m_playbin.installMessageFilter(filter: this); |
62 | |
63 | // Set the rest of the pipeline up |
64 | setAudioFlags(true); |
65 | |
66 | m_outputBin = QGstBin::create(name: "audio-output-bin"); |
67 | m_outputBin.add(ts: m_audioConvert); |
68 | |
69 | // add ghostpad |
70 | m_outputBin.addGhostPad(child: m_audioConvert, name: "sink"); |
71 | |
72 | g_object_set(object: m_playbin.object(), first_property_name: "audio-sink", m_outputBin.element(), NULL); |
73 | |
74 | // Set volume to 100% |
75 | gdouble volume = 1.0; |
76 | m_playbin.set(property: "volume", d: volume); |
77 | } |
78 | |
79 | QGstreamerAudioDecoder::~QGstreamerAudioDecoder() |
80 | { |
81 | stop(); |
82 | |
83 | m_playbin.removeMessageFilter(filter: this); |
84 | } |
85 | |
86 | bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message) |
87 | { |
88 | qCDebug(qLcGstreamerAudioDecoder) << "received bus message:"<< message; |
89 | |
90 | switch (message.type()) { |
91 | case GST_MESSAGE_DURATION: |
92 | return processBusMessageDuration(message); |
93 | |
94 | case GST_MESSAGE_ERROR: |
95 | return processBusMessageError(message); |
96 | |
97 | case GST_MESSAGE_WARNING: |
98 | return processBusMessageWarning(message); |
99 | |
100 | case GST_MESSAGE_INFO: |
101 | return processBusMessageInfo(message); |
102 | |
103 | case GST_MESSAGE_EOS: |
104 | return processBusMessageEOS(message); |
105 | |
106 | case GST_MESSAGE_STATE_CHANGED: |
107 | return processBusMessageStateChanged(message); |
108 | |
109 | case GST_MESSAGE_STREAMS_SELECTED: |
110 | return processBusMessageStreamsSelected(message); |
111 | |
112 | default: |
113 | return false; |
114 | } |
115 | } |
116 | |
117 | bool QGstreamerAudioDecoder::canReadQrc() const |
118 | { |
119 | return true; |
120 | } |
121 | |
122 | bool QGstreamerAudioDecoder::processBusMessageError(const QGstreamerMessage &message) |
123 | { |
124 | qCDebug(qLcGstreamerAudioDecoder) << " error"<< QCompactGstMessageAdaptor(message); |
125 | |
126 | QUniqueGErrorHandle err; |
127 | QGString debug; |
128 | gst_message_parse_error(message: message.message(), gerror: &err, debug: &debug); |
129 | |
130 | if (message.source() == m_playbin) { |
131 | if (err.get()->domain == GST_STREAM_ERROR |
132 | && err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) |
133 | processInvalidMedia(errorCode: QAudioDecoder::FormatError, |
134 | errorString: tr(s: "Cannot play stream of type: <unknown>")); |
135 | else |
136 | processInvalidMedia(errorCode: QAudioDecoder::ResourceError, |
137 | errorString: QString::fromUtf8(utf8: err.get()->message)); |
138 | } else { |
139 | QAudioDecoder::Error qerror = QAudioDecoder::ResourceError; |
140 | if (err.get()->domain == GST_STREAM_ERROR) { |
141 | switch (err.get()->code) { |
142 | case GST_STREAM_ERROR_DECRYPT: |
143 | case GST_STREAM_ERROR_DECRYPT_NOKEY: |
144 | qerror = QAudioDecoder::AccessDeniedError; |
145 | break; |
146 | case GST_STREAM_ERROR_FORMAT: |
147 | case GST_STREAM_ERROR_DEMUX: |
148 | case GST_STREAM_ERROR_DECODE: |
149 | case GST_STREAM_ERROR_WRONG_TYPE: |
150 | case GST_STREAM_ERROR_TYPE_NOT_FOUND: |
151 | case GST_STREAM_ERROR_CODEC_NOT_FOUND: |
152 | qerror = QAudioDecoder::FormatError; |
153 | break; |
154 | default: |
155 | break; |
156 | } |
157 | } else if (err.get()->domain == GST_CORE_ERROR) { |
158 | switch (err.get()->code) { |
159 | case GST_CORE_ERROR_MISSING_PLUGIN: |
160 | qerror = QAudioDecoder::FormatError; |
161 | break; |
162 | default: |
163 | break; |
164 | } |
165 | } |
166 | |
167 | processInvalidMedia(errorCode: qerror, errorString: QString::fromUtf8(utf8: err.get()->message)); |
168 | } |
169 | |
170 | return false; |
171 | } |
172 | |
173 | bool QGstreamerAudioDecoder::processBusMessageDuration(const QGstreamerMessage &) |
174 | { |
175 | updateDuration(); |
176 | return false; |
177 | } |
178 | |
179 | bool QGstreamerAudioDecoder::processBusMessageWarning(const QGstreamerMessage &message) |
180 | { |
181 | qCWarning(qLcGstreamerAudioDecoder) << "Warning:"<< QCompactGstMessageAdaptor(message); |
182 | return false; |
183 | } |
184 | |
185 | bool QGstreamerAudioDecoder::processBusMessageInfo(const QGstreamerMessage &message) |
186 | { |
187 | if (qLcGstreamerAudioDecoder().isDebugEnabled()) |
188 | qCWarning(qLcGstreamerAudioDecoder) << "Info:"<< QCompactGstMessageAdaptor(message); |
189 | return false; |
190 | } |
191 | |
192 | bool QGstreamerAudioDecoder::processBusMessageEOS(const QGstreamerMessage &) |
193 | { |
194 | m_playbin.setState(GST_STATE_NULL); |
195 | finished(); |
196 | return false; |
197 | } |
198 | |
199 | bool QGstreamerAudioDecoder::processBusMessageStateChanged(const QGstreamerMessage &message) |
200 | { |
201 | if (message.source() != m_playbin) |
202 | return false; |
203 | |
204 | GstState oldState; |
205 | GstState newState; |
206 | GstState pending; |
207 | |
208 | gst_message_parse_state_changed(message: message.message(), oldstate: &oldState, newstate: &newState, pending: &pending); |
209 | |
210 | bool isDecoding = false; |
211 | switch (newState) { |
212 | case GST_STATE_VOID_PENDING: |
213 | case GST_STATE_NULL: |
214 | case GST_STATE_READY: |
215 | break; |
216 | case GST_STATE_PLAYING: |
217 | isDecoding = true; |
218 | break; |
219 | case GST_STATE_PAUSED: |
220 | isDecoding = true; |
221 | |
222 | // gstreamer doesn't give a reliable indication the duration |
223 | // information is ready, GST_MESSAGE_DURATION is not sent by most elements |
224 | // the duration is queried up to 5 times with increasing delay |
225 | m_durationQueries = 5; |
226 | updateDuration(); |
227 | break; |
228 | } |
229 | |
230 | setIsDecoding(isDecoding); |
231 | return false; |
232 | } |
233 | |
234 | bool QGstreamerAudioDecoder::processBusMessageStreamsSelected(const QGstreamerMessage &message) |
235 | { |
236 | using namespace Qt::StringLiterals; |
237 | |
238 | QGstStreamCollectionHandle collection; |
239 | gst_message_parse_streams_selected(message: const_cast<GstMessage *>(message.message()), collection: &collection); |
240 | |
241 | bool hasAudio = false; |
242 | qForeachStreamInCollection(collection, f: [&](GstStream *stream) { |
243 | GstStreamType type = gst_stream_get_stream_type(stream); |
244 | if (type == GstStreamType::GST_STREAM_TYPE_AUDIO) |
245 | hasAudio = true; |
246 | }); |
247 | |
248 | if (!hasAudio) |
249 | processInvalidMedia(errorCode: QAudioDecoder::FormatError, errorString: u"No audio track in media"_s); |
250 | |
251 | return false; |
252 | } |
253 | |
254 | QUrl QGstreamerAudioDecoder::source() const |
255 | { |
256 | return mSource; |
257 | } |
258 | |
259 | void QGstreamerAudioDecoder::setSource(const QUrl &fileName) |
260 | { |
261 | stop(); |
262 | mDevice = nullptr; |
263 | |
264 | bool isSignalRequired = (mSource != fileName); |
265 | mSource = fileName; |
266 | if (isSignalRequired) |
267 | sourceChanged(); |
268 | } |
269 | |
270 | QIODevice *QGstreamerAudioDecoder::sourceDevice() const |
271 | { |
272 | return mDevice; |
273 | } |
274 | |
275 | void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device) |
276 | { |
277 | stop(); |
278 | mSource.clear(); |
279 | bool isSignalRequired = (mDevice != device); |
280 | mDevice = device; |
281 | if (isSignalRequired) |
282 | sourceChanged(); |
283 | } |
284 | |
285 | void QGstreamerAudioDecoder::start() |
286 | { |
287 | addAppSink(); |
288 | |
289 | if (!mSource.isEmpty()) { |
290 | m_playbin.set(property: "uri", str: mSource.toEncoded().constData()); |
291 | } else if (mDevice) { |
292 | // make sure we can read from device |
293 | if (!mDevice->isOpen() || !mDevice->isReadable()) { |
294 | processInvalidMedia(errorCode: QAudioDecoder::ResourceError, errorString: QLatin1String("Unable to read from specified device")); |
295 | return; |
296 | } |
297 | |
298 | QUrl streamURL = qGstRegisterQIODevice(mDevice); |
299 | m_playbin.set(property: "uri", str: streamURL.toEncoded().constData()); |
300 | } else { |
301 | return; |
302 | } |
303 | |
304 | // Set audio format |
305 | if (m_appSink) { |
306 | if (mFormat.isValid()) { |
307 | setAudioFlags(false); |
308 | auto caps = QGstUtils::capsForAudioFormat(format: mFormat); |
309 | m_appSink.setCaps(caps); |
310 | } else { |
311 | // We want whatever the native audio format is |
312 | setAudioFlags(true); |
313 | m_appSink.setCaps({}); |
314 | } |
315 | } |
316 | |
317 | if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { |
318 | qWarning() << "GStreamer; Unable to start decoding process"; |
319 | m_playbin.dumpGraph(fileNamePrefix: "failed"); |
320 | return; |
321 | } |
322 | } |
323 | |
324 | void QGstreamerAudioDecoder::stop() |
325 | { |
326 | m_playbin.setState(GST_STATE_NULL); |
327 | m_currentSessionId += 1; |
328 | removeAppSink(); |
329 | |
330 | // GStreamer thread is stopped. Can safely access m_buffersAvailable |
331 | if (m_buffersAvailable != 0) { |
332 | m_buffersAvailable = 0; |
333 | bufferAvailableChanged(available: false); |
334 | } |
335 | |
336 | if (m_position != invalidPosition) { |
337 | m_position = invalidPosition; |
338 | positionChanged(position: m_position.count()); |
339 | } |
340 | |
341 | if (m_duration != invalidDuration) { |
342 | m_duration = invalidDuration; |
343 | durationChanged(duration: m_duration.count()); |
344 | } |
345 | |
346 | setIsDecoding(false); |
347 | } |
348 | |
349 | QAudioFormat QGstreamerAudioDecoder::audioFormat() const |
350 | { |
351 | return mFormat; |
352 | } |
353 | |
354 | void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format) |
355 | { |
356 | if (mFormat != format) { |
357 | mFormat = format; |
358 | formatChanged(format: mFormat); |
359 | } |
360 | } |
361 | |
362 | QAudioBuffer QGstreamerAudioDecoder::read() |
363 | { |
364 | using namespace std::chrono; |
365 | |
366 | QAudioBuffer audioBuffer; |
367 | |
368 | if (m_buffersAvailable == 0) |
369 | return audioBuffer; |
370 | |
371 | m_buffersAvailable -= 1; |
372 | |
373 | if (m_buffersAvailable == 0) |
374 | bufferAvailableChanged(available: false); |
375 | |
376 | QGstSampleHandle sample = m_appSink.pullSample(); |
377 | GstBuffer *buffer = gst_sample_get_buffer(sample: sample.get()); |
378 | GstMapInfo mapInfo; |
379 | gst_buffer_map(buffer, info: &mapInfo, flags: GST_MAP_READ); |
380 | const char *bufferData = (const char *)mapInfo.data; |
381 | int bufferSize = mapInfo.size; |
382 | QAudioFormat format = QGstUtils::audioFormatForSample(sample: sample.get()); |
383 | |
384 | if (format.isValid()) { |
385 | // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. |
386 | // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. |
387 | nanoseconds position = getPositionFromBuffer(buffer); |
388 | audioBuffer = QAudioBuffer{ |
389 | QByteArray(bufferData, bufferSize), |
390 | format, |
391 | round<microseconds>(d: position).count(), |
392 | }; |
393 | milliseconds positionInMs = round<milliseconds>(d: position); |
394 | if (position != m_position) { |
395 | m_position = positionInMs; |
396 | positionChanged(position: m_position.count()); |
397 | } |
398 | } |
399 | gst_buffer_unmap(buffer, info: &mapInfo); |
400 | |
401 | return audioBuffer; |
402 | } |
403 | |
404 | qint64 QGstreamerAudioDecoder::position() const |
405 | { |
406 | return m_position.count(); |
407 | } |
408 | |
409 | qint64 QGstreamerAudioDecoder::duration() const |
410 | { |
411 | return m_duration.count(); |
412 | } |
413 | |
414 | void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) |
415 | { |
416 | stop(); |
417 | error(error: int(errorCode), errorString); |
418 | } |
419 | |
420 | GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *) |
421 | { |
422 | // "Note that the preroll buffer will also be returned as the first buffer when calling |
423 | // gst_app_sink_pull_buffer()." |
424 | |
425 | QMetaObject::invokeMethod(object: this, function: [this, sessionId = m_currentSessionId] { |
426 | if (sessionId != m_currentSessionId) |
427 | return; // stop()ed before message is executed |
428 | |
429 | m_buffersAvailable += 1; |
430 | bufferAvailableChanged(available: true); |
431 | bufferReady(); |
432 | }); |
433 | |
434 | return GST_FLOW_OK; |
435 | } |
436 | |
437 | GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data) |
438 | { |
439 | QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data); |
440 | qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample"; |
441 | return decoder->newSample(sink); |
442 | } |
443 | |
444 | void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) |
445 | { |
446 | int flags = m_playbin.getInt(property: "flags"); |
447 | // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired |
448 | // it prevents audio format conversion |
449 | flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO); |
450 | flags |= GST_PLAY_FLAG_AUDIO; |
451 | if (wantNativeAudio) |
452 | flags |= GST_PLAY_FLAG_NATIVE_AUDIO; |
453 | m_playbin.set(property: "flags", i: flags); |
454 | } |
455 | |
456 | void QGstreamerAudioDecoder::addAppSink() |
457 | { |
458 | using namespace std::chrono_literals; |
459 | |
460 | if (m_appSink) |
461 | return; |
462 | |
463 | qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink"; |
464 | m_appSink = QGstAppSink::create(name: "decoderAppSink"); |
465 | GstAppSinkCallbacks callbacks{}; |
466 | callbacks.new_sample = new_sample; |
467 | m_appSink.setCallbacks(callbacks, user_data: this, notify: nullptr); |
468 | |
469 | #if GST_CHECK_VERSION(1, 24, 0) |
470 | static constexpr auto maxBufferTime = 500ms; |
471 | m_appSink.setMaxBufferTime(maxBufferTime); |
472 | #else |
473 | static constexpr int maxBuffers = 16; |
474 | m_appSink.setMaxBuffers(maxBuffers); |
475 | #endif |
476 | |
477 | static constexpr bool sync = false; |
478 | m_appSink.setSync(sync); |
479 | |
480 | m_audioConvert.src().modifyPipelineInIdleProbe(f: [&] { |
481 | m_outputBin.add(ts: m_appSink); |
482 | qLinkGstElements(ts: m_audioConvert, ts: m_appSink); |
483 | }); |
484 | } |
485 | |
486 | void QGstreamerAudioDecoder::removeAppSink() |
487 | { |
488 | if (!m_appSink) |
489 | return; |
490 | |
491 | qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink"; |
492 | |
493 | m_audioConvert.src().modifyPipelineInIdleProbe(f: [&] { |
494 | qUnlinkGstElements(ts: m_audioConvert, ts: m_appSink); |
495 | m_outputBin.stopAndRemoveElements(ts&: m_appSink); |
496 | }); |
497 | |
498 | m_appSink = {}; |
499 | } |
500 | |
501 | void QGstreamerAudioDecoder::updateDuration() |
502 | { |
503 | std::optional<std::chrono::milliseconds> duration = m_playbin.durationInMs(); |
504 | if (!duration) |
505 | duration = invalidDuration; |
506 | |
507 | if (m_duration != duration) { |
508 | m_duration = *duration; |
509 | durationChanged(duration: m_duration.count()); |
510 | } |
511 | |
512 | if (m_duration.count() > 0) |
513 | m_durationQueries = 0; |
514 | |
515 | if (m_durationQueries > 0) { |
516 | //increase delay between duration requests |
517 | int delay = 25 << (5 - m_durationQueries); |
518 | QTimer::singleShot(interval: delay, receiver: this, slot: &QGstreamerAudioDecoder::updateDuration); |
519 | m_durationQueries--; |
520 | } |
521 | } |
522 | |
523 | std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer) |
524 | { |
525 | using namespace std::chrono; |
526 | using namespace std::chrono_literals; |
527 | nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) }; |
528 | if (position >= 0ns) |
529 | return position; |
530 | else |
531 | return invalidPosition; |
532 | } |
533 | |
534 | QT_END_NAMESPACE |
535 | |
536 | #include "moc_qgstreameraudiodecoder_p.cpp" |
537 |
Definitions
- qLcGstreamerAudioDecoder
- create
- QGstreamerAudioDecoder
- ~QGstreamerAudioDecoder
- processBusMessage
- canReadQrc
- processBusMessageError
- processBusMessageDuration
- processBusMessageWarning
- processBusMessageInfo
- processBusMessageEOS
- processBusMessageStateChanged
- processBusMessageStreamsSelected
- source
- setSource
- sourceDevice
- setSourceDevice
- start
- stop
- audioFormat
- setAudioFormat
- read
- position
- duration
- processInvalidMedia
- newSample
- new_sample
- setAudioFlags
- addAppSink
- removeAppSink
- updateDuration
Start learning QML with our Intro Training
Find out more