1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <common/qgstreamermediaplayer_p.h>
5
6#include <audio/qgstreameraudiodevice_p.h>
7#include <common/qglist_helper_p.h>
8#include <common/qgst_debug_p.h>
9#include <common/qgst_discoverer_p.h>
10#include <common/qgst_play_p.h>
11#include <common/qgstpipeline_p.h>
12#include <common/qgstreameraudiooutput_p.h>
13#include <common/qgstreamermessage_p.h>
14#include <common/qgstreamermetadata_p.h>
15#include <common/qgstreamervideooutput_p.h>
16#include <common/qgstreamervideosink_p.h>
17#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
18#include <qgstreamerformatinfo_p.h>
19
20#include <QtMultimedia/qaudiodevice.h>
21#include <QtCore/qdebug.h>
22#include <QtCore/qiodevice.h>
23#include <QtCore/qloggingcategory.h>
24#include <QtCore/qthread.h>
25#include <QtCore/qurl.h>
26#include <QtCore/private/quniquehandle_p.h>
27
28Q_STATIC_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player");
29
30QT_BEGIN_NAMESPACE
31
32namespace {
33
34std::optional<QGstreamerMediaPlayer::TrackType> toTrackType(const QGstCaps &caps)
35{
36 using TrackType = QGstreamerMediaPlayer::TrackType;
37
38 QByteArrayView type = caps.at(index: 0).name();
39
40 if (type.startsWith(other: "video/x-raw"))
41 return TrackType::VideoStream;
42 if (type.startsWith(other: "audio/x-raw"))
43 return TrackType::AudioStream;
44 if (type.startsWith(other: "text"))
45 return TrackType::SubtitleStream;
46
47 return std::nullopt;
48}
49
50} // namespace
51
52bool QGstreamerMediaPlayer::discover(const QUrl &url)
53{
54 QGst::QGstDiscoverer discoverer;
55
56 using namespace std::chrono;
57 using namespace std::chrono_literals;
58
59 auto discoveryResult = discoverer.discover(url);
60 if (discoveryResult) {
61 // Make sure GstPlay is ready if play() is called from slots during discovery
62 gst_play_set_uri(play: m_gstPlay.get(), uri: url.toEncoded().constData());
63
64 m_trackMetaData.fill(u: {});
65 seekableChanged(seekable: discoveryResult->isSeekable);
66 if (discoveryResult->duration)
67 m_duration = round<milliseconds>(d: *discoveryResult->duration);
68 else
69 m_duration = 0ms;
70 durationChanged(ms: m_duration);
71
72 m_metaData = QGst::toContainerMetadata(*discoveryResult);
73
74 videoAvailableChanged(videoAvailable: !discoveryResult->videoStreams.empty());
75 audioAvailableChanged(audioAvailable: !discoveryResult->audioStreams.empty());
76
77 m_nativeSize.clear();
78 for (const auto &videoInfo : discoveryResult->videoStreams) {
79 m_trackMetaData[0].emplace_back(args: QGst::toStreamMetadata(videoInfo));
80 QGstStructureView structure = videoInfo.caps.at(index: 0);
81 m_nativeSize.emplace_back(args: structure.nativeSize());
82 }
83 for (const auto &audioInfo : discoveryResult->audioStreams)
84 m_trackMetaData[1].emplace_back(args: QGst::toStreamMetadata(audioInfo));
85 for (const auto &subtitleInfo : discoveryResult->subtitleStreams)
86 m_trackMetaData[2].emplace_back(args: QGst::toStreamMetadata(subtitleInfo));
87
88 using Key = QMediaMetaData::Key;
89 auto copyKeysToRootMetadata = [&](const QMediaMetaData &reference, QSpan<const Key> keys) {
90 for (QMediaMetaData::Key key : keys) {
91 QVariant referenceValue = reference.value(k: key);
92 if (referenceValue.isValid())
93 m_metaData.insert(k: key, value: referenceValue);
94 }
95 };
96
97 // FIXME: we duplicate some metadata for the first audio / video track
98 // in future we will want to use e.g. the currently selected track
99 if (!m_trackMetaData[0].empty())
100 copyKeysToRootMetadata(m_trackMetaData[0].front(),
101 {
102 Key::HasHdrContent,
103 Key::Orientation,
104 Key::Resolution,
105 Key::VideoBitRate,
106 Key::VideoCodec,
107 Key::VideoFrameRate,
108 });
109
110 if (!m_trackMetaData[1].empty())
111 copyKeysToRootMetadata(m_trackMetaData[1].front(),
112 {
113 Key::AudioBitRate,
114 Key::AudioCodec,
115 });
116
117 if (!m_url.isEmpty())
118 m_metaData.insert(k: QMediaMetaData::Key::Url, value: m_url);
119
120 qCDebug(qLcMediaPlayer) << "metadata:" << m_metaData;
121 qCDebug(qLcMediaPlayer) << "video metadata:" << m_trackMetaData[0];
122 qCDebug(qLcMediaPlayer) << "audio metadata:" << m_trackMetaData[1];
123 qCDebug(qLcMediaPlayer) << "subtitle metadata:" << m_trackMetaData[2];
124
125 metaDataChanged();
126 tracksChanged();
127 m_activeTrack = {
128 isVideoAvailable() ? 0 : -1,
129 isAudioAvailable() ? 0 : -1,
130 -1,
131 };
132 updateVideoTrackEnabled();
133 updateAudioTrackEnabled();
134 updateNativeSizeOnVideoOutput();
135 }
136
137 return bool(discoveryResult);
138}
139
140void QGstreamerMediaPlayer::decoderPadAddedCustomSource(const QGstElement &src, const QGstPad &pad)
141{
142 // GStreamer or application thread
143 if (src != decoder)
144 return;
145
146 qCDebug(qLcMediaPlayer) << "Added pad" << pad.name() << "from" << src.name();
147
148 QGstCaps caps = pad.queryCaps();
149
150 std::optional<QGstreamerMediaPlayer::TrackType> type = toTrackType(caps);
151 if (!type)
152 return;
153
154 customPipelinePads[*type] = pad;
155
156 switch (*type) {
157 case VideoStream: {
158 QGstElement sink = gstVideoOutput->gstreamerVideoSink()
159 ? gstVideoOutput->gstreamerVideoSink()->gstSink()
160 : QGstElement::createFromPipelineDescription("fakesink");
161
162 customPipeline.add(ts: sink);
163 pad.link(sink: sink.sink());
164 customPipelineSinks[VideoStream] = sink;
165 sink.syncStateWithParent();
166 return;
167 }
168 case AudioStream: {
169 QGstElement sink = gstAudioOutput ? gstAudioOutput->gstElement()
170 : QGstElement::createFromPipelineDescription("fakesink");
171 customPipeline.add(ts: sink);
172 pad.link(sink: sink.sink());
173 customPipelineSinks[AudioStream] = sink;
174 sink.syncStateWithParent();
175 return;
176 }
177 case SubtitleStream: {
178 QGstElement sink = gstVideoOutput->gstreamerVideoSink()
179 ? gstVideoOutput->gstreamerVideoSink()->gstSink()
180 : QGstElement::createFromPipelineDescription("fakesink");
181 customPipeline.add(ts: sink);
182 pad.link(sink: sink.sink());
183 customPipelineSinks[SubtitleStream] = sink;
184 sink.syncStateWithParent();
185 return;
186 }
187
188 default:
189 Q_UNREACHABLE();
190 }
191}
192
193void QGstreamerMediaPlayer::decoderPadRemovedCustomSource(const QGstElement &src,
194 const QGstPad &pad)
195{
196 if (src != decoder)
197 return;
198
199 // application thread!
200 Q_ASSERT(thread()->isCurrentThread());
201
202 qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name() << "for stream"
203 << pad.streamId();
204
205 auto found = std::find(first: customPipelinePads.begin(), last: customPipelinePads.end(), val: pad);
206 if (found == customPipelinePads.end())
207 return;
208
209 TrackType type = TrackType(std::distance(first: customPipelinePads.begin(), last: found));
210
211 switch (type) {
212 case VideoStream:
213 case AudioStream:
214 case SubtitleStream: {
215 if (customPipelineSinks[VideoStream]) {
216 customPipeline.stopAndRemoveElements(ts&: customPipelineSinks[VideoStream]);
217 customPipelineSinks[VideoStream] = {};
218 }
219 return;
220
221 default:
222 Q_UNREACHABLE();
223 }
224 }
225}
226
227void QGstreamerMediaPlayer::resetStateForEmptyOrInvalidMedia()
228{
229 using namespace std::chrono_literals;
230 m_nativeSize.clear();
231
232 bool metadataNeedsSignal = !m_metaData.isEmpty();
233 bool tracksNeedsSignal =
234 std::any_of(first: m_trackMetaData.begin(), last: m_trackMetaData.end(), pred: [](const auto &container) {
235 return !container.empty();
236 });
237
238 m_metaData.clear();
239 m_trackMetaData.fill(u: {});
240 m_duration = 0ms;
241 seekableChanged(seekable: false);
242
243 videoAvailableChanged(videoAvailable: false);
244 audioAvailableChanged(audioAvailable: false);
245
246 m_activeTrack.fill(u: -1);
247
248 if (metadataNeedsSignal)
249 metaDataChanged();
250 if (tracksNeedsSignal)
251 tracksChanged();
252}
253
254void QGstreamerMediaPlayer::updateNativeSizeOnVideoOutput()
255{
256 int activeVideoTrack = activeTrack(TrackType::VideoStream);
257 bool hasVideoTrack = activeVideoTrack != -1;
258
259 QSize nativeSize = hasVideoTrack ? m_nativeSize[activeTrack(TrackType::VideoStream)] : QSize{};
260
261 QVariant orientation = hasVideoTrack
262 ? m_trackMetaData[TrackType::VideoStream][activeTrack(TrackType::VideoStream)].value(
263 k: QMediaMetaData::Key::Orientation)
264 : QVariant{};
265
266 if (orientation.isValid()) {
267 auto rotation = orientation.value<QtVideo::Rotation>();
268 gstVideoOutput->setRotation(rotation);
269 }
270 gstVideoOutput->setNativeSize(nativeSize);
271}
272
273void QGstreamerMediaPlayer::seekToCurrentPosition()
274{
275 gst_play_seek(play: m_gstPlay.get(), position: gst_play_get_position(play: m_gstPlay.get()));
276}
277
278void QGstreamerMediaPlayer::updateVideoTrackEnabled()
279{
280 bool hasTrack = m_activeTrack[TrackType::VideoStream] != -1;
281 bool hasSink = gstVideoOutput->gstreamerVideoSink() != nullptr;
282
283 gstVideoOutput->setActive(hasTrack);
284 gst_play_set_video_track_enabled(play: m_gstPlay.get(), enabled: hasTrack && hasSink);
285}
286
287void QGstreamerMediaPlayer::updateAudioTrackEnabled()
288{
289 bool hasTrack = m_activeTrack[TrackType::AudioStream] != -1;
290 bool hasAudioOut = gstAudioOutput;
291
292 gst_play_set_audio_track_enabled(play: m_gstPlay.get(), enabled: hasTrack && hasAudioOut);
293}
294
295void QGstreamerMediaPlayer::updateBufferProgress(float newProgress)
296{
297 if (qFuzzyIsNull(f: newProgress - m_bufferProgress))
298 return;
299
300 m_bufferProgress = newProgress;
301 bufferProgressChanged(progress: m_bufferProgress);
302}
303
304void QGstreamerMediaPlayer::disconnectDecoderHandlers()
305{
306 auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ &sourceSetup };
307 for (QGObjectHandlerScopedConnection *handler : handlers)
308 handler->disconnect();
309}
310
311q23::expected<QPlatformMediaPlayer *, QString> QGstreamerMediaPlayer::create(QMediaPlayer *parent)
312{
313 auto videoOutput = QGstreamerVideoOutput::create();
314 if (!videoOutput)
315 return q23::unexpected{ videoOutput.error() };
316
317 return new QGstreamerMediaPlayer(videoOutput.value(), parent);
318}
319
320template <typename T>
321void setSeekAccurate(T *config, gboolean accurate)
322{
323 gst_play_config_set_seek_accurate(config, accurate);
324}
325
326QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput,
327 QMediaPlayer *parent)
328 : QObject(parent),
329 QPlatformMediaPlayer(parent),
330 gstVideoOutput(videoOutput),
331 m_gstPlay{
332 gst_play_new(video_renderer: nullptr),
333 QGstPlayHandle::HasRef,
334 },
335 m_playbin{
336 GST_PIPELINE_CAST(gst_play_get_pipeline(m_gstPlay.get())),
337 QGstPipeline::HasRef,
338 },
339 m_gstPlayBus{
340 QGstBusHandle{ gst_play_get_message_bus(play: m_gstPlay.get()), QGstBusHandle::HasRef },
341 }
342{
343#if 1
344 // LATER: remove this hack after meta-freescale decides not to pull in outdated APIs
345
346 // QTBUG-131300: nxp deliberately reverted to an old gst-play API before the gst-play API
347 // stabilized. compare:
348 // https://github.com/nxp-imx/gst-plugins-bad/commit/ff04fa9ca1b79c98e836d8cdb26ac3502dafba41
349 constexpr bool useNxpWorkaround = std::is_same_v<decltype(&gst_play_config_set_seek_accurate),
350 void (*)(GstPlay *, gboolean)>;
351
352 QUniqueGstStructureHandle config{
353 gst_play_get_config(play: m_gstPlay.get()),
354 };
355
356 if constexpr (useNxpWorkaround)
357 setSeekAccurate(config: m_gstPlay.get(), accurate: true);
358 else
359 setSeekAccurate(config: config.get(), accurate: true);
360
361 gst_play_set_config(play: m_gstPlay.get(), config: config.release());
362#else
363 QUniqueGstStructureHandle config{
364 gst_play_get_config(m_gstPlay.get()),
365 };
366 gst_play_config_set_seek_accurate(config.get(), true);
367 gst_play_set_config(m_gstPlay.get(), config.release());
368#endif
369
370 gstVideoOutput->setParent(this);
371
372 m_playbin.set(property: "video-sink", o: gstVideoOutput->gstElement());
373 m_playbin.set(property: "text-sink", o: gstVideoOutput->gstSubtitleElement());
374 m_playbin.set(property: "audio-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
375
376 m_gstPlayBus.installMessageFilter(this);
377
378 // we start without subtitles
379 gst_play_set_subtitle_track_enabled(play: m_gstPlay.get(), enabled: false);
380
381 sourceSetup = m_playbin.connect(name: "source-setup", callback: GCallback(sourceSetupCallback), userData: this);
382
383 m_activeTrack.fill(u: -1);
384
385 // TODO: how to detect stalled media?
386}
387
388QGstreamerMediaPlayer::~QGstreamerMediaPlayer()
389{
390 if (customPipeline)
391 cleanupCustomPipeline();
392
393 m_gstPlayBus.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this));
394 gst_bus_set_flushing(bus: m_gstPlayBus.get(), TRUE);
395 gst_play_stop(play: m_gstPlay.get());
396
397 // NOTE: gst_play_stop is not sufficient, un-reffing m_gstPlay can deadlock
398 m_playbin.setStateSync(state: GST_STATE_NULL);
399
400 m_playbin.set(property: "video-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
401 m_playbin.set(property: "text-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
402 m_playbin.set(property: "audio-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
403}
404
405void QGstreamerMediaPlayer::updatePositionFromPipeline()
406{
407 using namespace std::chrono;
408
409 positionChanged(ms: round<milliseconds>(d: nanoseconds{
410 gst_play_get_position(play: m_gstPlay.get()),
411 }));
412}
413
414bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message)
415{
416 if (isCustomSource()) {
417 constexpr bool traceBusMessages = true;
418 if (traceBusMessages)
419 qCDebug(qLcMediaPlayer) << "received bus message:" << message;
420
421 switch (message.type()) {
422 case GST_MESSAGE_WARNING:
423 qWarning() << "received bus message:" << message;
424 break;
425
426 case GST_MESSAGE_INFO:
427 qInfo() << "received bus message:" << message;
428 break;
429
430 case GST_MESSAGE_ERROR:
431 qWarning() << "received bus message:" << message;
432 customPipeline.dumpPipelineGraph(filename: "GST_MESSAGE_ERROR");
433 break;
434
435 case GST_MESSAGE_LATENCY:
436 customPipeline.recalculateLatency();
437 break;
438
439 default:
440 break;
441 }
442 return false;
443 }
444
445 switch (message.type()) {
446 case GST_MESSAGE_APPLICATION:
447 if (gst_play_is_play_message(msg: message.message()))
448 return processBusMessageApplication(message);
449 return false;
450
451 default:
452 qCDebug(qLcMediaPlayer) << message;
453
454 return false;
455 }
456
457 return false;
458}
459
460bool QGstreamerMediaPlayer::processBusMessageApplication(const QGstreamerMessage &message)
461{
462 using namespace std::chrono;
463 GstPlayMessage type;
464 gst_play_message_parse_type(msg: message.message(), type: &type);
465 qCDebug(qLcMediaPlayer) << QGstPlayMessageAdaptor{ message };
466
467 switch (type) {
468 case GST_PLAY_MESSAGE_URI_LOADED: {
469 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
470 return false;
471 }
472
473 case GST_PLAY_MESSAGE_POSITION_UPDATED: {
474 if (state() == QMediaPlayer::PlaybackState::PlayingState) {
475
476 constexpr bool usePayload = false;
477 if constexpr (usePayload) {
478 GstClockTime position;
479 gst_play_message_parse_position_updated(msg: message.message(), position: &position);
480 positionChanged(ms: round<milliseconds>(d: nanoseconds{ position }));
481 } else {
482 GstClockTime position = gst_play_get_position(play: m_gstPlay.get());
483 positionChanged(ms: round<milliseconds>(d: nanoseconds{ position }));
484 }
485 }
486 return false;
487 }
488 case GST_PLAY_MESSAGE_DURATION_CHANGED: {
489 GstClockTime duration;
490 gst_play_message_parse_duration_updated(msg: message.message(), duration: &duration);
491 milliseconds durationInMs = round<milliseconds>(d: nanoseconds{ duration });
492 durationChanged(ms: durationInMs);
493
494 m_metaData.insert(k: QMediaMetaData::Duration, value: int(durationInMs.count()));
495 metaDataChanged();
496
497 return false;
498 }
499 case GST_PLAY_MESSAGE_BUFFERING: {
500 guint percent;
501 gst_play_message_parse_buffering_percent(msg: message.message(), percent: &percent);
502 updateBufferProgress(newProgress: percent * 0.01f);
503 return false;
504 }
505 case GST_PLAY_MESSAGE_STATE_CHANGED: {
506 GstPlayState state;
507 gst_play_message_parse_state_changed(msg: message.message(), state: &state);
508
509 switch (state) {
510 case GstPlayState::GST_PLAY_STATE_STOPPED:
511 if (stateChangeToSkip) {
512 qCDebug(qLcMediaPlayer) << " skipping StoppedState transition";
513
514 stateChangeToSkip -= 1;
515 return false;
516 }
517 stateChanged(newState: QMediaPlayer::StoppedState);
518 updateBufferProgress(newProgress: 0);
519 return false;
520
521 case GstPlayState::GST_PLAY_STATE_PAUSED:
522 stateChanged(newState: QMediaPlayer::PausedState);
523 mediaStatusChanged(status: QMediaPlayer::BufferedMedia);
524 gstVideoOutput->setActive(true);
525 updateBufferProgress(newProgress: 1);
526 return false;
527 case GstPlayState::GST_PLAY_STATE_BUFFERING:
528 mediaStatusChanged(status: QMediaPlayer::BufferingMedia);
529 return false;
530 case GstPlayState::GST_PLAY_STATE_PLAYING:
531 stateChanged(newState: QMediaPlayer::PlayingState);
532 mediaStatusChanged(status: QMediaPlayer::BufferedMedia);
533 gstVideoOutput->setActive(true);
534 updateBufferProgress(newProgress: 1);
535
536 return false;
537 default:
538 return false;
539 }
540 }
541 case GST_PLAY_MESSAGE_MEDIA_INFO_UPDATED: {
542 using namespace QGstPlaySupport;
543
544 QUniqueGstPlayMediaInfoHandle info{};
545 gst_play_message_parse_media_info_updated(msg: message.message(), info: &info);
546
547 seekableChanged(seekable: gst_play_media_info_is_seekable(info: info.get()));
548
549 const gchar *title = gst_play_media_info_get_title(info: info.get());
550 m_metaData.insert(k: QMediaMetaData::Title, value: QString::fromUtf8(utf8: title));
551
552 metaDataChanged();
553 tracksChanged();
554
555 return false;
556 }
557 case GST_PLAY_MESSAGE_END_OF_STREAM: {
558 if (doLoop()) {
559 positionChanged(ms: m_duration);
560 qCDebug(qLcMediaPlayer) << "EOS: restarting loop";
561 gst_play_play(play: m_gstPlay.get());
562 positionChanged(ms: 0ms);
563
564 // we will still get a GST_PLAY_MESSAGE_STATE_CHANGED message, which we will just ignore
565 // for now
566 stateChangeToSkip += 1;
567 } else {
568 qCDebug(qLcMediaPlayer) << "EOS: done";
569 positionChanged(ms: m_duration);
570 mediaStatusChanged(status: QMediaPlayer::EndOfMedia);
571 stateChanged(newState: QMediaPlayer::StoppedState);
572 gstVideoOutput->setActive(false);
573 }
574
575 return false;
576 }
577 case GST_PLAY_MESSAGE_ERROR:
578 case GST_PLAY_MESSAGE_WARNING:
579 case GST_PLAY_MESSAGE_VIDEO_DIMENSIONS_CHANGED:
580 case GST_PLAY_MESSAGE_VOLUME_CHANGED:
581 case GST_PLAY_MESSAGE_MUTE_CHANGED:
582 case GST_PLAY_MESSAGE_SEEK_DONE:
583 return false;
584
585 default:
586 Q_UNREACHABLE_RETURN(false);
587 }
588}
589
590qint64 QGstreamerMediaPlayer::duration() const
591{
592 return m_duration.count();
593}
594
595bool QGstreamerMediaPlayer::hasMedia() const
596{
597 return !m_url.isEmpty() || m_stream;
598}
599
600bool QGstreamerMediaPlayer::hasValidMedia() const
601{
602 if (!hasMedia())
603 return false;
604
605 switch (mediaStatus()) {
606 case QMediaPlayer::MediaStatus::NoMedia:
607 case QMediaPlayer::MediaStatus::InvalidMedia:
608 return false;
609
610 default:
611 return true;
612 }
613}
614
615float QGstreamerMediaPlayer::bufferProgress() const
616{
617 return m_bufferProgress;
618}
619
620QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const
621{
622 return QMediaTimeRange();
623}
624
625qreal QGstreamerMediaPlayer::playbackRate() const
626{
627 return gst_play_get_rate(play: m_gstPlay.get());
628}
629
630void QGstreamerMediaPlayer::setPlaybackRate(qreal rate)
631{
632 if (isCustomSource()) {
633 static std::once_flag flag;
634 std::call_once(once&: flag, f: [] {
635 // CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
636 qWarning()
637 << "setPlaybackRate with custom gstreamer pipelines can cause pipeline hangs. "
638 "Use with care";
639 });
640
641 customPipeline.setPlaybackRate(rate);
642 return;
643 }
644
645 if (rate == playbackRate())
646 return;
647
648 qCDebug(qLcMediaPlayer) << "gst_play_set_rate" << rate;
649 gst_play_set_rate(play: m_gstPlay.get(), rate);
650 playbackRateChanged(rate);
651}
652
653void QGstreamerMediaPlayer::setPosition(qint64 pos)
654{
655 std::chrono::milliseconds posInMs{ pos };
656
657 setPosition(posInMs);
658}
659
660void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos)
661{
662 using namespace std::chrono;
663
664 if (isCustomSource()) {
665 static std::once_flag flag;
666 std::call_once(once&: flag, f: [] {
667 // CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
668 qWarning() << "setPosition with custom gstreamer pipelines can cause pipeline hangs. "
669 "Use with care";
670 });
671
672 customPipeline.setPosition(pos);
673 return;
674 } else {
675 qCDebug(qLcMediaPlayer) << "gst_play_seek" << pos;
676 gst_play_seek(play: m_gstPlay.get(), position: nanoseconds(pos).count());
677
678 if (mediaStatus() == QMediaPlayer::EndOfMedia)
679 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
680 }
681 positionChanged(ms: pos);
682}
683
684void QGstreamerMediaPlayer::play()
685{
686 if (isCustomSource()) {
687 gstVideoOutput->setActive(true);
688 customPipeline.setState(GST_STATE_PLAYING);
689 stateChanged(newState: QMediaPlayer::PlayingState);
690 return;
691 }
692
693 QMediaPlayer::PlaybackState currentState = state();
694 if (currentState == QMediaPlayer::PlayingState || !hasValidMedia())
695 return;
696
697 if (currentState != QMediaPlayer::PausedState)
698 resetCurrentLoop();
699
700 if (mediaStatus() == QMediaPlayer::EndOfMedia) {
701 positionChanged(position: 0);
702 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
703 }
704
705 if (m_pendingSeek) {
706 gst_play_seek(play: m_gstPlay.get(), position: m_pendingSeek->count());
707 m_pendingSeek = std::nullopt;
708 }
709
710 qCDebug(qLcMediaPlayer) << "gst_play_play";
711 gstVideoOutput->setActive(true);
712 gst_play_play(play: m_gstPlay.get());
713 stateChanged(newState: QMediaPlayer::PlayingState);
714}
715
716void QGstreamerMediaPlayer::pause()
717{
718 if (isCustomSource()) {
719 gstVideoOutput->setActive(true);
720 customPipeline.setState(GST_STATE_PAUSED);
721 stateChanged(newState: QMediaPlayer::PausedState);
722 return;
723 }
724
725 if (state() == QMediaPlayer::PausedState || !hasMedia()
726 || m_resourceErrorState != ResourceErrorState::NoError)
727 return;
728
729 gstVideoOutput->setActive(true);
730
731 qCDebug(qLcMediaPlayer) << "gst_play_pause";
732 gst_play_pause(play: m_gstPlay.get());
733
734 mediaStatusChanged(status: QMediaPlayer::BufferedMedia);
735 stateChanged(newState: QMediaPlayer::PausedState);
736}
737
738void QGstreamerMediaPlayer::stop()
739{
740 if (isCustomSource()) {
741 customPipeline.setState(GST_STATE_READY);
742 stateChanged(newState: QMediaPlayer::StoppedState);
743 gstVideoOutput->setActive(false);
744 return;
745 }
746
747 using namespace std::chrono_literals;
748 if (state() == QMediaPlayer::StoppedState) {
749 if (position() != 0) {
750 m_pendingSeek = 0ms;
751 positionChanged(ms: 0ms);
752 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
753 }
754 return;
755 }
756
757 qCDebug(qLcMediaPlayer) << "gst_play_stop";
758 gstVideoOutput->setActive(false);
759 gst_play_stop(play: m_gstPlay.get());
760
761 stateChanged(newState: QMediaPlayer::StoppedState);
762
763 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
764 positionChanged(ms: 0ms);
765}
766
767const QGstPipeline &QGstreamerMediaPlayer::pipeline() const
768{
769 if (isCustomSource())
770 return customPipeline;
771
772 return m_playbin;
773}
774
775bool QGstreamerMediaPlayer::canPlayQrc() const
776{
777 return true;
778}
779
780bool QGstreamerMediaPlayer::pitchCompensation() const
781{
782 return true;
783}
784
785QPlatformMediaPlayer::PitchCompensationAvailability
786QGstreamerMediaPlayer::pitchCompensationAvailability() const
787{
788 return PitchCompensationAvailability::AlwaysOn;
789}
790
791QUrl QGstreamerMediaPlayer::media() const
792{
793 return m_url;
794}
795
796const QIODevice *QGstreamerMediaPlayer::mediaStream() const
797{
798 return m_stream;
799}
800
801void QGstreamerMediaPlayer::sourceSetupCallback([[maybe_unused]] GstElement *playbin,
802 GstElement *source, QGstreamerMediaPlayer *)
803{
804 // gst_play thread
805
806 const gchar *typeName = g_type_name_from_instance(instance: (GTypeInstance *)source);
807 qCDebug(qLcMediaPlayer) << "Setting up source:" << typeName;
808
809 if (typeName == std::string_view("GstRTSPSrc")) {
810 QGstElement s(source, QGstElement::NeedsRef);
811 int latency{40};
812 bool ok{false};
813 int v = qEnvironmentVariableIntValue(varName: "QT_MEDIA_RTSP_LATENCY", ok: &ok);
814 if (ok)
815 latency = v;
816 qCDebug(qLcMediaPlayer) << " -> setting source latency to:" << latency << "ms";
817 s.set(property: "latency", i: latency);
818
819 bool drop{true};
820 v = qEnvironmentVariableIntValue(varName: "QT_MEDIA_RTSP_DROP_ON_LATENCY", ok: &ok);
821 if (ok && v == 0)
822 drop = false;
823 qCDebug(qLcMediaPlayer) << " -> setting drop-on-latency to:" << drop;
824 s.set(property: "drop-on-latency", b: drop);
825
826 bool retrans{false};
827 v = qEnvironmentVariableIntValue(varName: "QT_MEDIA_RTSP_DO_RETRANSMISSION", ok: &ok);
828 if (ok && v != 0)
829 retrans = true;
830 qCDebug(qLcMediaPlayer) << " -> setting do-retransmission to:" << retrans;
831 s.set(property: "do-retransmission", b: retrans);
832 }
833}
834
835void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream)
836{
837 using namespace Qt::Literals;
838 using namespace std::chrono;
839 using namespace std::chrono_literals;
840
841 if (customPipeline)
842 cleanupCustomPipeline();
843
844 m_resourceErrorState = ResourceErrorState::NoError;
845 m_url = content;
846 m_stream = stream;
847 QUrl streamURL;
848 if (stream)
849 streamURL = qGstRegisterQIODevice(stream);
850
851 if (content.isEmpty() && !stream) {
852 mediaStatusChanged(status: QMediaPlayer::NoMedia);
853 resetStateForEmptyOrInvalidMedia();
854 return;
855 }
856
857 if (isCustomSource()) {
858 setMediaCustomSource(content);
859 } else {
860 mediaStatusChanged(status: QMediaPlayer::LoadingMedia);
861 const QUrl &playUrl = stream ? streamURL : content;
862
863 // LATER: discover is synchronous, but we would be way more friendly to make it
864 // asynchronous.
865 bool mediaDiscovered = discover(url: playUrl);
866 if (!mediaDiscovered) {
867 m_resourceErrorState = ResourceErrorState::ErrorOccurred;
868 error(QMediaPlayer::Error::ResourceError, errorString: u"Resource cannot be discovered"_s);
869 mediaStatusChanged(status: QMediaPlayer::InvalidMedia);
870 resetStateForEmptyOrInvalidMedia();
871 return;
872 }
873
874 positionChanged(ms: 0ms);
875 }
876}
877
878void QGstreamerMediaPlayer::setMediaCustomSource(const QUrl &content)
879{
880 using namespace Qt::Literals;
881 using namespace std::chrono;
882 using namespace std::chrono_literals;
883
884 {
885 // FIXME: claim sinks
886 // TODO: move ownership of sinks to gst_play after using them
887 m_playbin.set(property: "video-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
888 m_playbin.set(property: "text-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
889 m_playbin.set(property: "audio-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
890
891 if (gstVideoOutput->gstreamerVideoSink()) {
892 if (QGstElement sink = gstVideoOutput->gstreamerVideoSink()->gstSink())
893 sink.removeFromParent();
894 }
895 }
896
897 customPipeline = QGstPipeline::create(name: "customPipeline");
898 customPipeline.installMessageFilter(filter: this);
899 positionUpdateTimer = std::make_unique<QTimer>();
900
901 QObject::connect(sender: positionUpdateTimer.get(), signal: &QTimer::timeout, context: this, slot: [this] {
902 Q_ASSERT(customPipeline);
903 auto position = customPipeline.position();
904
905 positionChanged(ms: round<milliseconds>(d: position));
906 });
907
908 positionUpdateTimer->start(value: 100ms);
909
910 QByteArray gstLaunchString =
911 content.toString(options: QUrl::RemoveScheme | QUrl::PrettyDecoded).toLatin1();
912 qCDebug(qLcMediaPlayer) << "generating" << gstLaunchString;
913 QGstElement element = QGstElement::createFromPipelineDescription(gstLaunchString);
914 if (!element) {
915 emit error(QMediaPlayer::ResourceError, errorString: u"Could not create custom pipeline"_s);
916 return;
917 }
918
919 decoder = element;
920 customPipeline.add(ts: decoder);
921
922 QGstBin elementBin{
923 qGstSafeCast<GstBin>(arg: element.element()),
924 QGstBin::NeedsRef,
925 };
926 if (elementBin) // bins are expected to provide unconnected src pads
927 elementBin.addUnlinkedGhostPads(GstPadDirection::GST_PAD_SRC);
928
929 // for all other elements
930 padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAddedCustomSource>(instance: this);
931 padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemovedCustomSource>(instance: this);
932
933 customPipeline.setStateSync(state: GstState::GST_STATE_PAUSED);
934
935 auto srcPadVisitor = [](GstElement *element, GstPad *pad, void *self) -> gboolean {
936 reinterpret_cast<QGstreamerMediaPlayer *>(self)->decoderPadAddedCustomSource(
937 src: QGstElement{ element, QGstElement::NeedsRef }, pad: QGstPad{ pad, QGstPad::NeedsRef });
938 return true;
939 };
940
941 gst_element_foreach_pad(element: element.element(), func: srcPadVisitor, user_data: this);
942
943 mediaStatusChanged(status: QMediaPlayer::LoadedMedia);
944
945 customPipeline.dumpGraph(fileNamePrefix: "setMediaCustomPipeline");
946}
947
948void QGstreamerMediaPlayer::cleanupCustomPipeline()
949{
950 customPipeline.setStateSync(state: GST_STATE_NULL);
951 customPipeline.removeMessageFilter(filter: this);
952
953 for (QGstElement &sink : customPipelineSinks)
954 if (sink)
955 customPipeline.remove(ts: sink);
956
957 positionUpdateTimer = {};
958 customPipeline = {};
959}
960
961void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
962{
963 if (isCustomSource()) {
964 qWarning() << "QMediaPlayer::setAudioOutput not supported when using custom sources";
965 return;
966 }
967
968 if (gstAudioOutput == output)
969 return;
970
971 auto *gstOutput = static_cast<QGstreamerAudioOutput *>(output);
972 if (gstOutput)
973 gstOutput->setAsync(true);
974
975 gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
976 if (gstAudioOutput)
977 m_playbin.set(property: "audio-sink", o: gstAudioOutput->gstElement());
978 else
979 m_playbin.set(property: "audio-sink", o: QGstElement::createFromPipelineDescription("fakesink"));
980 updateAudioTrackEnabled();
981
982 // FIXME: we need to have a gst_play API to change the sinks on the fly.
983 // finishStateChange a hack to avoid assertion failures in gstreamer
984 if (!qmediaplayerDestructorCalled)
985 m_playbin.finishStateChange();
986}
987
988QMediaMetaData QGstreamerMediaPlayer::metaData() const
989{
990 return m_metaData;
991}
992
993void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink)
994{
995 if (isCustomSource()) {
996 qWarning() << "QMediaPlayer::setVideoSink not supported when using custom sources";
997 return;
998 }
999
1000 auto *gstSink = sink ? static_cast<QGstreamerVideoSink *>(sink->platformVideoSink()) : nullptr;
1001 if (gstSink)
1002 gstSink->setAsync(false);
1003
1004 gstVideoOutput->setVideoSink(sink);
1005 updateVideoTrackEnabled();
1006
1007 if (sink && state() == QMediaPlayer::PausedState) {
1008 // FIXME: we want to get a the existing frame, but gst_play does not have such capabilities.
1009 // seeking to the current position is a rather bad hack, but it's the best we can do for now
1010 seekToCurrentPosition();
1011 }
1012}
1013
1014int QGstreamerMediaPlayer::trackCount(QPlatformMediaPlayer::TrackType type)
1015{
1016 QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
1017 return tracks.size();
1018}
1019
1020QMediaMetaData QGstreamerMediaPlayer::trackMetaData(QPlatformMediaPlayer::TrackType type, int index)
1021{
1022 QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
1023 if (index < tracks.size())
1024 return tracks[index];
1025 return {};
1026}
1027
1028int QGstreamerMediaPlayer::activeTrack(TrackType type)
1029{
1030 return m_activeTrack[type];
1031}
1032
1033void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index)
1034{
1035 if (m_activeTrack[type] == index)
1036 return;
1037
1038 int formerTrack = m_activeTrack[type];
1039 m_activeTrack[type] = index;
1040
1041 switch (type) {
1042 case TrackType::VideoStream: {
1043 if (index != -1)
1044 gst_play_set_video_track(play: m_gstPlay.get(), stream_index: index);
1045 updateVideoTrackEnabled();
1046 updateNativeSizeOnVideoOutput();
1047 break;
1048 }
1049 case TrackType::AudioStream: {
1050 if (index != -1)
1051 gst_play_set_audio_track(play: m_gstPlay.get(), stream_index: index);
1052 updateAudioTrackEnabled();
1053 break;
1054 }
1055 case TrackType::SubtitleStream: {
1056 if (index != -1)
1057 gst_play_set_subtitle_track(play: m_gstPlay.get(), stream_index: index);
1058 gst_play_set_subtitle_track_enabled(play: m_gstPlay.get(), enabled: index != -1);
1059 break;
1060 }
1061 default:
1062 Q_UNREACHABLE();
1063 };
1064
1065 if (formerTrack != -1 && index != -1)
1066 // it can take several seconds for gstreamer to switch the track. so we seek to the current
1067 // position
1068 seekToCurrentPosition();
1069}
1070
1071QT_END_NAMESPACE
1072

source code of qtmultimedia/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp