1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "playbackengine/qffmpegstreamdecoder_p.h"
5#include "playbackengine/qffmpegmediadataholder_p.h"
6#include <qloggingcategory.h>
7
8QT_BEGIN_NAMESPACE
9
10static Q_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder");
11
12namespace QFFmpeg {
13
14StreamDecoder::StreamDecoder(const Codec &codec, qint64 absSeekPos)
15 : m_codec(codec),
16 m_absSeekPos(absSeekPos),
17 m_trackType(MediaDataHolder::trackTypeFromMediaType(mediaType: codec.context()->codec_type))
18{
19 qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType
20 << "absSeekPos:" << absSeekPos;
21 Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes);
22}
23
24StreamDecoder::~StreamDecoder()
25{
26 avcodec_flush_buffers(avctx: m_codec.context());
27}
28
29void StreamDecoder::onFinalPacketReceived()
30{
31 decode({});
32}
33
34void StreamDecoder::setInitialPosition(TimePoint, qint64 trackPos)
35{
36 m_absSeekPos = trackPos;
37}
38
39void StreamDecoder::decode(Packet packet)
40{
41 m_packets.enqueue(t: packet);
42
43 scheduleNextStep();
44}
45
46void StreamDecoder::doNextStep()
47{
48 auto packet = m_packets.dequeue();
49
50 auto decodePacket = [this](Packet packet) {
51 if (trackType() == QPlatformMediaPlayer::SubtitleStream)
52 decodeSubtitle(packet);
53 else
54 decodeMedia(packet);
55 };
56
57 if (packet.isValid() && packet.loopOffset().index != m_offset.index) {
58 decodePacket({});
59
60 qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:" << packet.loopOffset().index;
61
62 avcodec_flush_buffers(avctx: m_codec.context());
63 m_offset = packet.loopOffset();
64 }
65
66 decodePacket(packet);
67
68 setAtEnd(!packet.isValid());
69
70 if (packet.isValid())
71 emit packetProcessed(packet);
72
73 scheduleNextStep(allowDoImmediatelly: false);
74}
75
76QPlatformMediaPlayer::TrackType StreamDecoder::trackType() const
77{
78 return m_trackType;
79}
80
81qint32 StreamDecoder::maxQueueSize(QPlatformMediaPlayer::TrackType type)
82{
83 switch (type) {
84
85 case QPlatformMediaPlayer::VideoStream:
86 return 3;
87 case QPlatformMediaPlayer::AudioStream:
88 return 9;
89 case QPlatformMediaPlayer::SubtitleStream:
90 return 6; /*main packet and closing packet*/
91 default:
92 Q_UNREACHABLE_RETURN(-1);
93 }
94}
95
96void StreamDecoder::onFrameProcessed(Frame frame)
97{
98 if (frame.sourceId() != id())
99 return;
100
101 --m_pendingFramesCount;
102 Q_ASSERT(m_pendingFramesCount >= 0);
103
104 scheduleNextStep();
105}
106
107bool StreamDecoder::canDoNextStep() const
108{
109 const qint32 maxCount = maxQueueSize(type: m_trackType);
110
111 return !m_packets.empty() && m_pendingFramesCount < maxCount
112 && PlaybackEngineObject::canDoNextStep();
113}
114
115void StreamDecoder::onFrameFound(Frame frame)
116{
117 if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos)
118 return;
119
120 Q_ASSERT(m_pendingFramesCount >= 0);
121 ++m_pendingFramesCount;
122 emit requestHandleFrame(frame);
123}
124
125void StreamDecoder::decodeMedia(Packet packet)
126{
127 auto sendPacketResult = sendAVPacket(packet);
128
129 if (sendPacketResult == AVERROR(EAGAIN)) {
130 // Doc says:
131 // AVERROR(EAGAIN): input is not accepted in the current state - user
132 // must read output with avcodec_receive_frame() (once
133 // all output is read, the packet should be resent, and
134 // the call will not fail with EAGAIN).
135 receiveAVFrames();
136 sendPacketResult = sendAVPacket(packet);
137
138 if (sendPacketResult != AVERROR(EAGAIN))
139 qWarning() << "Unexpected FFmpeg behavior";
140 }
141
142 if (sendPacketResult == 0)
143 receiveAVFrames(flushPacket: !packet.isValid());
144}
145
146int StreamDecoder::sendAVPacket(Packet packet)
147{
148 return avcodec_send_packet(avctx: m_codec.context(), avpkt: packet.isValid() ? packet.avPacket() : nullptr);
149}
150
151void StreamDecoder::receiveAVFrames(bool flushPacket)
152{
153 while (true) {
154 auto avFrame = makeAVFrame();
155
156 const auto receiveFrameResult = avcodec_receive_frame(avctx: m_codec.context(), frame: avFrame.get());
157
158 if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) {
159 if (flushPacket && receiveFrameResult == AVERROR(EAGAIN)) {
160 // The documentation says that in the EAGAIN state output is not available. The new
161 // input must be sent. It does not say that this state can also be returned for
162 // Android MediaCodec when the ff_AMediaCodec_dequeueOutputBuffer call times out.
163 // The flush packet means it is the end of the stream. No more packets are available,
164 // so getting EAGAIN is unexpected here. At this point, the EAGAIN status was probably
165 // caused by a timeout in the ffmpeg implementation, not by too few packets. That is
166 // why there will be another try of calling avcodec_receive_frame
167 qWarning() << "Unexpected FFmpeg behavior: EAGAIN state for avcodec_receive_frame "
168 << "at end of the stream";
169 flushPacket = false;
170 continue;
171 }
172 break;
173 }
174
175 if (receiveFrameResult < 0) {
176 emit error(code: QMediaPlayer::FormatError, errorString: err2str(errnum: receiveFrameResult));
177 break;
178 }
179
180
181 // Avoid starvation on FFmpeg decoders with fixed size frame pool
182 if (m_trackType == QPlatformMediaPlayer::VideoStream)
183 avFrame = copyFromHwPool(frame: std::move(avFrame));
184
185 onFrameFound(frame: { m_offset, std::move(avFrame), m_codec, 0, id() });
186 }
187}
188
189void StreamDecoder::decodeSubtitle(Packet packet)
190{
191 if (!packet.isValid())
192 return;
193 // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" <<
194 // (codec->codec->capabilities & AV_CODEC_CAP_DELAY);
195 AVSubtitle subtitle;
196 memset(s: &subtitle, c: 0, n: sizeof(subtitle));
197 int gotSubtitle = 0;
198
199 const int res =
200 avcodec_decode_subtitle2(avctx: m_codec.context(), sub: &subtitle, got_sub_ptr: &gotSubtitle, avpkt: packet.avPacket());
201 // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format <<
202 // Qt::hex << (quint64)subtitle.pts;
203 if (res < 0 || !gotSubtitle)
204 return;
205
206 // apparently the timestamps in the AVSubtitle structure are not always filled in
207 // if they are missing, use the packets pts and duration values instead
208 qint64 start, end;
209 if (subtitle.pts == AV_NOPTS_VALUE) {
210 start = m_codec.toUs(ts: packet.avPacket()->pts);
211 end = start + m_codec.toUs(ts: packet.avPacket()->duration);
212 } else {
213 auto pts = timeStampUs(ts: subtitle.pts, base: AVRational{ .num: 1, AV_TIME_BASE });
214 start = *pts + qint64(subtitle.start_display_time) * 1000;
215 end = *pts + qint64(subtitle.end_display_time) * 1000;
216 }
217
218 if (end <= start) {
219 qWarning() << "Invalid subtitle time";
220 return;
221 }
222 // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):";
223 QString text;
224 for (uint i = 0; i < subtitle.num_rects; ++i) {
225 const auto *r = subtitle.rects[i];
226 // qCDebug(qLcDecoder) << " subtitletext:" << r->text << "/" << r->ass;
227 if (i)
228 text += QLatin1Char('\n');
229 if (r->text)
230 text += QString::fromUtf8(utf8: r->text);
231 else {
232 const char *ass = r->ass;
233 int nCommas = 0;
234 while (*ass) {
235 if (nCommas == 8)
236 break;
237 if (*ass == ',')
238 ++nCommas;
239 ++ass;
240 }
241 text += QString::fromUtf8(utf8: ass);
242 }
243 }
244 text.replace(before: QLatin1String("\\N"), after: QLatin1String("\n"));
245 text.replace(before: QLatin1String("\\n"), after: QLatin1String("\n"));
246 text.replace(before: QLatin1String("\r\n"), after: QLatin1String("\n"));
247 if (text.endsWith(c: QLatin1Char('\n')))
248 text.chop(n: 1);
249
250 onFrameFound(frame: { m_offset, text, start, end - start, id() });
251
252 // TODO: maybe optimize
253 onFrameFound(frame: { m_offset, QString(), end, 0, id() });
254}
255} // namespace QFFmpeg
256
257QT_END_NAMESPACE
258
259#include "moc_qffmpegstreamdecoder_p.cpp"
260

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp