1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "playbackengine/qffmpegstreamdecoder_p.h"
5#include "playbackengine/qffmpegmediadataholder_p.h"
6#include <qloggingcategory.h>
7
8QT_BEGIN_NAMESPACE
9
10static Q_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder");
11
12namespace QFFmpeg {
13
14StreamDecoder::StreamDecoder(const CodecContext &codecContext, TrackPosition absSeekPos)
15 : m_codecContext(codecContext),
16 m_absSeekPos(absSeekPos),
17 m_trackType(MediaDataHolder::trackTypeFromMediaType(mediaType: codecContext.context()->codec_type))
18{
19 qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType
20 << "absSeekPos:" << absSeekPos.get();
21 Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes);
22}
23
24StreamDecoder::~StreamDecoder()
25{
26 avcodec_flush_buffers(avctx: m_codecContext.context());
27}
28
29void StreamDecoder::onFinalPacketReceived()
30{
31 decode({});
32}
33
34void StreamDecoder::decode(Packet packet)
35{
36 m_packets.enqueue(t: packet);
37
38 scheduleNextStep();
39}
40
41void StreamDecoder::doNextStep()
42{
43 auto packet = m_packets.dequeue();
44
45 auto decodePacket = [this](Packet packet) {
46 if (trackType() == QPlatformMediaPlayer::SubtitleStream)
47 decodeSubtitle(packet);
48 else
49 decodeMedia(packet);
50 };
51
52 if (packet.isValid() && packet.loopOffset().loopIndex != m_offset.loopIndex) {
53 decodePacket({});
54
55 qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:"
56 << packet.loopOffset().loopIndex;
57
58 avcodec_flush_buffers(avctx: m_codecContext.context());
59 m_offset = packet.loopOffset();
60 }
61
62 decodePacket(packet);
63
64 setAtEnd(!packet.isValid());
65
66 if (packet.isValid())
67 emit packetProcessed(packet);
68
69 scheduleNextStep(allowDoImmediatelly: false);
70}
71
72QPlatformMediaPlayer::TrackType StreamDecoder::trackType() const
73{
74 return m_trackType;
75}
76
77qint32 StreamDecoder::maxQueueSize(QPlatformMediaPlayer::TrackType type)
78{
79 switch (type) {
80
81 case QPlatformMediaPlayer::VideoStream:
82 return 3;
83 case QPlatformMediaPlayer::AudioStream:
84 return 9;
85 case QPlatformMediaPlayer::SubtitleStream:
86 return 6; /*main packet and closing packet*/
87 default:
88 Q_UNREACHABLE_RETURN(-1);
89 }
90}
91
92void StreamDecoder::onFrameProcessed(Frame frame)
93{
94 if (frame.sourceId() != id())
95 return;
96
97 --m_pendingFramesCount;
98 Q_ASSERT(m_pendingFramesCount >= 0);
99
100 scheduleNextStep();
101}
102
103bool StreamDecoder::canDoNextStep() const
104{
105 const qint32 maxCount = maxQueueSize(type: m_trackType);
106
107 return !m_packets.empty() && m_pendingFramesCount < maxCount
108 && PlaybackEngineObject::canDoNextStep();
109}
110
111void StreamDecoder::onFrameFound(Frame frame)
112{
113 if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos)
114 return;
115
116 Q_ASSERT(m_pendingFramesCount >= 0);
117 ++m_pendingFramesCount;
118 emit requestHandleFrame(frame);
119}
120
121void StreamDecoder::decodeMedia(Packet packet)
122{
123 auto sendPacketResult = sendAVPacket(packet);
124
125 if (sendPacketResult == AVERROR(EAGAIN)) {
126 // Doc says:
127 // AVERROR(EAGAIN): input is not accepted in the current state - user
128 // must read output with avcodec_receive_frame() (once
129 // all output is read, the packet should be resent, and
130 // the call will not fail with EAGAIN).
131 receiveAVFrames();
132 sendPacketResult = sendAVPacket(packet);
133
134 if (sendPacketResult != AVERROR(EAGAIN))
135 qWarning() << "Unexpected FFmpeg behavior";
136 }
137
138 if (sendPacketResult == 0)
139 receiveAVFrames(flushPacket: !packet.isValid());
140}
141
142int StreamDecoder::sendAVPacket(Packet packet)
143{
144 return avcodec_send_packet(avctx: m_codecContext.context(), avpkt: packet.isValid() ? packet.avPacket() : nullptr);
145}
146
147void StreamDecoder::receiveAVFrames(bool flushPacket)
148{
149 while (true) {
150 auto avFrame = makeAVFrame();
151
152 const auto receiveFrameResult = avcodec_receive_frame(avctx: m_codecContext.context(), frame: avFrame.get());
153
154 if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) {
155 if (flushPacket && receiveFrameResult == AVERROR(EAGAIN)) {
156 // The documentation says that in the EAGAIN state output is not available. The new
157 // input must be sent. It does not say that this state can also be returned for
158 // Android MediaCodec when the ff_AMediaCodec_dequeueOutputBuffer call times out.
159 // The flush packet means it is the end of the stream. No more packets are available,
160 // so getting EAGAIN is unexpected here. At this point, the EAGAIN status was probably
161 // caused by a timeout in the ffmpeg implementation, not by too few packets. That is
162 // why there will be another try of calling avcodec_receive_frame
163 qWarning() << "Unexpected FFmpeg behavior: EAGAIN state for avcodec_receive_frame "
164 << "at end of the stream";
165 flushPacket = false;
166 continue;
167 }
168 break;
169 }
170
171 if (receiveFrameResult < 0) {
172 emit error(code: QMediaPlayer::FormatError, errorString: err2str(errnum: receiveFrameResult));
173 break;
174 }
175
176
177 // Avoid starvation on FFmpeg decoders with fixed size frame pool
178 if (m_trackType == QPlatformMediaPlayer::VideoStream)
179 avFrame = copyFromHwPool(frame: std::move(avFrame));
180
181 onFrameFound(frame: { m_offset, std::move(avFrame), m_codecContext, id() });
182 }
183}
184
185void StreamDecoder::decodeSubtitle(Packet packet)
186{
187 if (!packet.isValid())
188 return;
189 // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" <<
190 // (codec->codec->capabilities & AV_CODEC_CAP_DELAY);
191 AVSubtitle subtitle;
192 memset(s: &subtitle, c: 0, n: sizeof(subtitle));
193 int gotSubtitle = 0;
194
195 const int res =
196 avcodec_decode_subtitle2(avctx: m_codecContext.context(), sub: &subtitle, got_sub_ptr: &gotSubtitle, avpkt: packet.avPacket());
197 // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format <<
198 // Qt::hex << (quint64)subtitle.pts;
199 if (res < 0 || !gotSubtitle)
200 return;
201
202 // apparently the timestamps in the AVSubtitle structure are not always filled in
203 // if they are missing, use the packets pts and duration values instead
204 TrackPosition start = 0, end = 0;
205 if (subtitle.pts == AV_NOPTS_VALUE) {
206 start = m_codecContext.toTrackPosition(streamPosition: AVStreamPosition(packet.avPacket()->pts));
207 end = start + m_codecContext.toTrackDuration(duration: AVStreamDuration(packet.avPacket()->duration));
208 } else {
209 auto pts = timeStampUs(ts: subtitle.pts, base: AVRational{ .num: 1, AV_TIME_BASE });
210 start = TrackPosition(*pts + qint64(subtitle.start_display_time) * 1000);
211 end = TrackPosition(*pts + qint64(subtitle.end_display_time) * 1000);
212 }
213
214 if (end <= start) {
215 qWarning() << "Invalid subtitle time";
216 return;
217 }
218 // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):";
219 QString text;
220 for (uint i = 0; i < subtitle.num_rects; ++i) {
221 const auto *r = subtitle.rects[i];
222 // qCDebug(qLcDecoder) << " subtitletext:" << r->text << "/" << r->ass;
223 if (i)
224 text += QLatin1Char('\n');
225 if (r->text)
226 text += QString::fromUtf8(utf8: r->text);
227 else {
228 const char *ass = r->ass;
229 int nCommas = 0;
230 while (*ass) {
231 if (nCommas == 8)
232 break;
233 if (*ass == ',')
234 ++nCommas;
235 ++ass;
236 }
237 text += QString::fromUtf8(utf8: ass);
238 }
239 }
240 text.replace(before: QLatin1String("\\N"), after: QLatin1String("\n"));
241 text.replace(before: QLatin1String("\\n"), after: QLatin1String("\n"));
242 text.replace(before: QLatin1String("\r\n"), after: QLatin1String("\n"));
243 if (text.endsWith(c: QLatin1Char('\n')))
244 text.chop(n: 1);
245
246 onFrameFound(frame: { m_offset, text, start, end - start, id() });
247
248 // TODO: maybe optimize
249 onFrameFound(frame: { m_offset, QString(), end, TrackDuration(0), id() });
250}
251} // namespace QFFmpeg
252
253QT_END_NAMESPACE
254
255#include "moc_qffmpegstreamdecoder_p.cpp"
256

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp