1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "playbackengine/qffmpegdemuxer_p.h"
5#include <qloggingcategory.h>
6#include <chrono>
7
8QT_BEGIN_NAMESPACE
9
10namespace QFFmpeg {
11
12// 4 sec for buffering. TODO: maybe move to env var customization
13static constexpr TrackDuration MaxBufferedDurationUs{ 4'000'000 };
14
15// around 4 sec of hdr video
16static constexpr qint64 MaxBufferedSize = 32 * 1024 * 1024;
17
18Q_STATIC_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer");
19
20static TrackPosition packetEndPos(const Packet &packet, const AVStream *stream,
21 const AVFormatContext *context)
22{
23 const AVPacket &avPacket = *packet.avPacket();
24 return packet.loopOffset().loopStartTimeUs.asDuration()
25 + toTrackPosition(streamPosition: AVStreamPosition(avPacket.pts + avPacket.duration), avStream: stream, formatContext: context);
26}
27
28static bool isPacketWithinStreamDuration(const AVFormatContext *context, const Packet &packet)
29{
30 const AVPacket &avPacket = *packet.avPacket();
31 const AVStream &avStream = *context->streams[avPacket.stream_index];
32 const AVStreamDuration streamDuration(avStream.duration);
33 if (streamDuration.get() <= 0
34 || context->duration_estimation_method != AVFMT_DURATION_FROM_STREAM)
35 return true; // Stream duration shouldn't or doesn't need to be compared to pts
36
37 if (avPacket.pts == AV_NOPTS_VALUE) { // Unexpected situation
38 qWarning() << "QFFmpeg::Demuxer received AVPacket with pts == AV_NOPTS_VALUE";
39 return true;
40 }
41
42 if (avStream.start_time != AV_NOPTS_VALUE)
43 return AVStreamDuration(avPacket.pts - avStream.start_time) <= streamDuration;
44
45 const TrackPosition trackPos = toTrackPosition(streamPosition: AVStreamPosition(avPacket.pts), avStream: &avStream, formatContext: context);
46 const TrackPosition trackPosOfStreamEnd = toTrackDuration(streamDuration, avStream: &avStream).asTimePoint();
47 return trackPos <= trackPosOfStreamEnd;
48
49 // TODO: If there is a packet that starts before the canonical end of the stream but has a
50 // malformed duration, rework doNextStep to check for eof after that packet.
51}
52
53Demuxer::Demuxer(AVFormatContext *context, TrackPosition initialPosUs, bool seekPending,
54 const LoopOffset &loopOffset, const StreamIndexes &streamIndexes, int loops)
55 : m_context(context),
56 m_seeked(!seekPending && initialPosUs == TrackPosition{ 0 }), // Don't seek to 0 unless seek requested
57 m_posInLoopUs{ initialPosUs },
58 m_loopOffset(loopOffset),
59 m_loops(loops)
60{
61 qCDebug(qLcDemuxer) << "Create demuxer."
62 << "pos:" << m_posInLoopUs.get()
63 << "loop offset:" << m_loopOffset.loopStartTimeUs.get()
64 << "loop index:" << m_loopOffset.loopIndex << "loops:" << loops;
65
66 Q_ASSERT(m_context);
67
68 for (auto i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) {
69 if (streamIndexes[i] >= 0) {
70 const auto trackType = static_cast<QPlatformMediaPlayer::TrackType>(i);
71 qCDebug(qLcDemuxer) << "Activate demuxing stream" << i << ", trackType:" << trackType;
72 m_streams[streamIndexes[i]] = { .trackType: trackType };
73 }
74 }
75}
76
77void Demuxer::doNextStep()
78{
79 ensureSeeked();
80
81 Packet packet(m_loopOffset, AVPacketUPtr{ av_packet_alloc() }, id());
82 AVPacket &avPacket = *packet.avPacket();
83
84 const int demuxStatus = av_read_frame(s: m_context, pkt: &avPacket);
85
86 const int streamIndex = avPacket.stream_index;
87 auto streamIterator = m_streams.find(x: streamIndex);
88 const bool streamIsRelevant = streamIterator != m_streams.end();
89
90 if (demuxStatus == AVERROR_EOF
91 || (streamIsRelevant && !isPacketWithinStreamDuration(context: m_context, packet))) {
92 ++m_loopOffset.loopIndex;
93
94 const auto loops = m_loops.loadAcquire();
95 if (loops >= 0 && m_loopOffset.loopIndex >= loops) {
96 qCDebug(qLcDemuxer) << "finish demuxing";
97
98 if (!std::exchange(obj&: m_buffered, new_val: true))
99 emit packetsBuffered();
100
101 setAtEnd(true);
102 } else {
103 // start next loop
104 m_seeked = false;
105 m_posInLoopUs = TrackPosition(0);
106 m_loopOffset.loopStartTimeUs = m_maxPacketsEndPos;
107 m_maxPacketsEndPos = TrackPosition(0);
108
109 ensureSeeked();
110
111 qCDebug(qLcDemuxer) << "Demuxer loops changed. Index:" << m_loopOffset.loopIndex
112 << "Offset:" << m_loopOffset.loopStartTimeUs.get();
113
114 scheduleNextStep(allowDoImmediatelly: false);
115 }
116
117 return;
118 }
119
120 if (demuxStatus < 0) {
121 qCWarning(qLcDemuxer) << "Demuxing failed" << demuxStatus << AVError(demuxStatus);
122
123 if (demuxStatus == AVERROR(EAGAIN) && m_demuxerRetryCount != s_maxDemuxerRetries) {
124 // When demuxer reports EAGAIN, we can try to recover by calling av_read_frame again.
125 // The documentation for av_read_frame does not mention this, but FFmpeg command line
126 // tool does this, see input_thread() function in ffmpeg_demux.c. There, the response
127 // is to sleep for 10 ms before trying again. NOTE: We do not have any known way of
128 // reproducing this in our tests.
129 ++m_demuxerRetryCount;
130
131 qCDebug(qLcDemuxer) << "Retrying";
132 scheduleNextStep(allowDoImmediatelly: false);
133 } else {
134 // av_read_frame reports another error. This could for example happen if network is
135 // disconnected while playing a network stream, where av_read_frame may return
136 // ETIMEDOUT.
137 // TODO: Demuxer errors should likely stop playback in media player examples.
138 emit error(QMediaPlayer::ResourceError,
139 errorString: QLatin1StringView("Demuxing failed"));
140 }
141
142 return;
143 }
144
145 m_demuxerRetryCount = 0;
146
147 if (streamIsRelevant) {
148 auto &streamData = streamIterator->second;
149 const AVStream *stream = m_context->streams[streamIndex];
150
151 const TrackPosition endPos = packetEndPos(packet, stream, context: m_context);
152 m_maxPacketsEndPos = qMax(a: m_maxPacketsEndPos, b: endPos);
153
154 // Increase buffered metrics as the packet has been processed.
155
156 streamData.bufferedDuration += toTrackDuration(streamDuration: AVStreamDuration(avPacket.duration), avStream: stream);
157 streamData.bufferedSize += avPacket.size;
158 streamData.maxSentPacketsPos = qMax(a: streamData.maxSentPacketsPos, b: endPos);
159 updateStreamDataLimitFlag(streamData);
160
161 if (!m_buffered && streamData.isDataLimitReached) {
162 m_buffered = true;
163 emit packetsBuffered();
164 }
165
166 if (!m_firstPacketFound) {
167 m_firstPacketFound = true;
168 emit firstPacketFound(id: id(), absSeekPos: m_posInLoopUs + m_loopOffset.loopStartTimeUs.asDuration());
169 }
170
171 auto signal = signalByTrackType(trackType: streamData.trackType);
172 emit (this->*signal)(packet);
173 }
174
175 scheduleNextStep(allowDoImmediatelly: false);
176}
177
178void Demuxer::onPacketProcessed(Packet packet)
179{
180 Q_ASSERT(packet.isValid());
181
182 if (packet.sourceId() != id())
183 return;
184
185 auto &avPacket = *packet.avPacket();
186
187 const auto streamIndex = avPacket.stream_index;
188 const auto stream = m_context->streams[streamIndex];
189 auto it = m_streams.find(x: streamIndex);
190
191 if (it != m_streams.end()) {
192 auto &streamData = it->second;
193
194 // Decrease buffered metrics as new data (the packet) has been received (buffered)
195
196 streamData.bufferedDuration -= toTrackDuration(streamDuration: AVStreamDuration(avPacket.duration), avStream: stream);
197 streamData.bufferedSize -= avPacket.size;
198 streamData.maxProcessedPacketPos =
199 qMax(a: streamData.maxProcessedPacketPos, b: packetEndPos(packet, stream, context: m_context));
200
201 Q_ASSERT(it->second.bufferedDuration >= TrackDuration(0));
202 Q_ASSERT(it->second.bufferedSize >= 0);
203
204 updateStreamDataLimitFlag(streamData);
205 }
206
207 scheduleNextStep();
208}
209
210std::chrono::milliseconds Demuxer::timerInterval() const
211{
212 using namespace std::chrono_literals;
213 return m_demuxerRetryCount != 0 ? s_demuxerRetryInterval : PlaybackEngineObject::timerInterval();
214}
215
216bool Demuxer::canDoNextStep() const
217{
218 auto isDataLimitReached = [](const auto &streamIndexToData) {
219 return streamIndexToData.second.isDataLimitReached;
220 };
221
222 // Demuxer waits:
223 // - if it's paused
224 // - if the end has been reached
225 // - if streams are empty (probably, should be handled on the initialization)
226 // - if at least one of the streams has reached the data limit (duration or size)
227
228 return PlaybackEngineObject::canDoNextStep() && !isAtEnd() && !m_streams.empty()
229 && std::none_of(first: m_streams.begin(), last: m_streams.end(), pred: isDataLimitReached);
230}
231
232void Demuxer::ensureSeeked()
233{
234 if (std::exchange(obj&: m_seeked, new_val: true))
235 return;
236
237 if ((m_context->ctx_flags & AVFMTCTX_UNSEEKABLE) == 0) {
238
239 // m_posInLoopUs is intended to be the number of microseconds since playback start, and is
240 // in the range [0, duration()]. av_seek_frame seeks to a position relative to the start of
241 // the media timeline, which may be non-zero. We adjust for this by adding the
242 // AVFormatContext's start_time.
243 //
244 // NOTE: m_posInLoop is not calculated correctly if the start_time is non-zero, but
245 // this must be fixed separately.
246 const AVContextPosition seekPos = toContextPosition(trackPosition: m_posInLoopUs, formatContext: m_context);
247
248 qCDebug(qLcDemuxer).nospace()
249 << "Seeking to offset " << m_posInLoopUs.get() << "us from media start.";
250
251 auto err = av_seek_frame(s: m_context, stream_index: -1, timestamp: seekPos.get(), AVSEEK_FLAG_BACKWARD);
252
253 if (err < 0) {
254 qCWarning(qLcDemuxer) << "Failed to seek, pos" << seekPos.get();
255
256 // Drop an error of seeking to initial position of streams with undefined duration.
257 // This needs improvements.
258 if (m_posInLoopUs != TrackPosition{ 0 } || m_context->duration > 0)
259 emit error(QMediaPlayer::ResourceError,
260 errorString: QLatin1StringView("Failed to seek: ") + err2str(errnum: err));
261 }
262 }
263
264 setAtEnd(false);
265}
266
267Demuxer::RequestingSignal Demuxer::signalByTrackType(QPlatformMediaPlayer::TrackType trackType)
268{
269 switch (trackType) {
270 case QPlatformMediaPlayer::TrackType::VideoStream:
271 return &Demuxer::requestProcessVideoPacket;
272 case QPlatformMediaPlayer::TrackType::AudioStream:
273 return &Demuxer::requestProcessAudioPacket;
274 case QPlatformMediaPlayer::TrackType::SubtitleStream:
275 return &Demuxer::requestProcessSubtitlePacket;
276 default:
277 Q_ASSERT(!"Unknown track type");
278 }
279
280 return nullptr;
281}
282
283void Demuxer::setLoops(int loopsCount)
284{
285 qCDebug(qLcDemuxer) << "setLoops to demuxer" << loopsCount;
286 m_loops.storeRelease(newValue: loopsCount);
287}
288
289void Demuxer::updateStreamDataLimitFlag(StreamData &streamData)
290{
291 const TrackDuration packetsPosDiff =
292 streamData.maxSentPacketsPos - streamData.maxProcessedPacketPos;
293 streamData.isDataLimitReached = streamData.bufferedDuration >= MaxBufferedDurationUs
294 || (streamData.bufferedDuration == TrackDuration(0)
295 && packetsPosDiff >= MaxBufferedDurationUs)
296 || streamData.bufferedSize >= MaxBufferedSize;
297}
298
299} // namespace QFFmpeg
300
301QT_END_NAMESPACE
302
303#include "moc_qffmpegdemuxer_p.cpp"
304

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp