1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3#include "qffmpegrecordingengine_p.h"
4#include "qffmpegencodinginitializer_p.h"
5#include "qffmpegaudioencoder_p.h"
6#include "qffmpegaudioinput_p.h"
7#include "qffmpegrecordingengineutils_p.h"
8
9#include "private/qmultimediautils_p.h"
10#include "private/qplatformaudiobufferinput_p.h"
11#include "private/qplatformvideosource_p.h"
12#include "private/qplatformvideoframeinput_p.h"
13
14#include "qdebug.h"
15#include "qffmpegvideoencoder_p.h"
16#include "qffmpegmediametadata_p.h"
17#include "qffmpegmuxer_p.h"
18#include "qloggingcategory.h"
19
20QT_BEGIN_NAMESPACE
21
22Q_STATIC_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder");
23
24namespace QFFmpeg
25{
26
27RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings,
28 std::unique_ptr<EncodingFormatContext> context)
29 : m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this))
30{
31 Q_ASSERT(m_formatContext);
32 Q_ASSERT(m_formatContext->isAVIOOpen());
33}
34
35RecordingEngine::~RecordingEngine()
36{
37 Q_ASSERT(m_state == State::Finalizing);
38}
39
40void RecordingEngine::addAudioInput(QFFmpegAudioInput *input)
41{
42 Q_ASSERT(input);
43 Q_ASSERT(m_state == State::FormatsInitializing);
44
45 if (input->device.isNull()) {
46 emit streamInitializationError(code: QMediaRecorder::ResourceError,
47 description: QLatin1StringView("Audio device is null"));
48 return;
49 }
50
51 const QAudioFormat format = input->device.preferredFormat();
52
53 if (!format.isValid()) {
54 emit streamInitializationError(
55 code: QMediaRecorder::FormatError,
56 description: QLatin1StringView("Audio device has invalid preferred format"));
57 return;
58 }
59
60 AudioEncoder *audioEncoder = createAudioEncoder(format);
61 connectEncoderToSource(encoder: audioEncoder, source: input);
62}
63
64void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input,
65 const QAudioBuffer &firstBuffer)
66{
67 Q_ASSERT(input);
68 Q_ASSERT(m_state == State::FormatsInitializing);
69
70 const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat();
71
72 AudioEncoder *audioEncoder = createAudioEncoder(format);
73
74 // set the buffer before connecting to avoid potential races
75 if (firstBuffer.isValid())
76 audioEncoder->addBuffer(buffer: firstBuffer);
77
78 connectEncoderToSource(encoder: audioEncoder, source: input);
79}
80
81AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format)
82{
83 Q_ASSERT(format.isValid());
84
85 auto audioEncoder = new AudioEncoder(*this, format, m_settings);
86
87 m_audioEncoders.emplace_back(args&: audioEncoder);
88 connect(sender: audioEncoder, signal: &EncoderThread::endOfSourceStream, context: this,
89 slot: &RecordingEngine::handleSourceEndOfStream);
90 connect(sender: audioEncoder, signal: &EncoderThread::initialized, context: this,
91 slot: &RecordingEngine::handleEncoderInitialization, type: Qt::SingleShotConnection);
92 if (m_autoStop)
93 audioEncoder->setAutoStop(true);
94
95 return audioEncoder;
96}
97
98void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame)
99{
100 Q_ASSERT(m_state == State::FormatsInitializing);
101
102 QVideoFrameFormat frameFormat =
103 firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat();
104
105 Q_ASSERT(frameFormat.isValid());
106
107 if (firstFrame.isValid() && frameFormat.streamFrameRate() <= 0.f) {
108 const qint64 startTime = firstFrame.startTime();
109 const qint64 endTime = firstFrame.endTime();
110 if (startTime != -1 && endTime > startTime)
111 frameFormat.setStreamFrameRate(static_cast<qreal>(VideoFrameTimeBase)
112 / (endTime - startTime));
113 }
114
115 std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat()
116 ? AVPixelFormat(*source->ffmpegHWPixelFormat())
117 : std::optional<AVPixelFormat>{};
118
119 qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":"
120 << "pixelFormat=" << frameFormat.pixelFormat()
121 << "frameSize=" << frameFormat.frameSize()
122 << "frameRate=" << frameFormat.streamFrameRate()
123 << "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE);
124
125 auto videoEncoder = new VideoEncoder(*this, m_settings, frameFormat, hwPixelFormat);
126 m_videoEncoders.emplace_back(args&: videoEncoder);
127 if (m_autoStop)
128 videoEncoder->setAutoStop(true);
129
130 connect(sender: videoEncoder, signal: &EncoderThread::endOfSourceStream, context: this,
131 slot: &RecordingEngine::handleSourceEndOfStream);
132
133 connect(sender: videoEncoder, signal: &EncoderThread::initialized, context: this,
134 slot: &RecordingEngine::handleEncoderInitialization, type: Qt::SingleShotConnection);
135
136 // set the frame before connecting to avoid potential races
137 if (firstFrame.isValid())
138 videoEncoder->addFrame(frame: firstFrame);
139
140 connectEncoderToSource(encoder: videoEncoder, source);
141}
142
143bool RecordingEngine::startEncoders()
144{
145 Q_ASSERT(m_state == State::FormatsInitializing);
146 Q_ASSERT(m_formatsInitializer);
147 m_formatsInitializer.reset();
148
149 if (m_audioEncoders.empty() && m_videoEncoders.empty()) {
150 emit sessionError(code: QMediaRecorder::ResourceError,
151 description: QLatin1StringView("No valid stream found for encoding"));
152 return false;
153 }
154
155 m_state = State::EncodersInitializing;
156
157 forEachEncoder(f: [](EncoderThread *encoder) { //
158 encoder->start();
159 });
160
161 return true;
162}
163
164bool RecordingEngine::initialize(const std::vector<QAudioBufferSource *> &audioSources,
165 const std::vector<QPlatformVideoSource *> &videoSources)
166{
167 Q_ASSERT(m_state == State::None);
168
169 m_state = State::FormatsInitializing;
170 m_formatsInitializer = std::make_unique<EncodingInitializer>(args&: *this);
171 return m_formatsInitializer->start(audioSources, videoSources);
172}
173
174RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine,
175 bool writeTrailer)
176 : m_recordingEngine(recordingEngine), m_writeTrailer(writeTrailer)
177{
178 setObjectName(QStringLiteral("EncodingFinalizer"));
179 Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
180 connect(sender: this, signal: &QThread::finished, context: this, slot: &QObject::deleteLater);
181}
182
183void RecordingEngine::EncodingFinalizer::run()
184{
185 Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
186
187 m_recordingEngine.stopAndDeleteThreads();
188
189 if (m_writeTrailer) {
190 const int res = av_write_trailer(s: m_recordingEngine.avFormatContext());
191 if (res < 0) {
192 qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << AVError(res);
193 emit m_recordingEngine.sessionError(code: QMediaRecorder::FormatError,
194 description: QLatin1String("Cannot write trailer: ")
195 + err2str(errnum: res));
196 }
197 }
198 // else ffmpeg might crash
199
200 // close AVIO before emitting finalizationDone.
201 m_recordingEngine.m_formatContext->closeAVIO();
202
203 qCDebug(qLcFFmpegEncoder) << "Media recording finalized";
204 emit m_recordingEngine.finalizationDone();
205 auto recordingEnginePtr = &m_recordingEngine;
206 delete recordingEnginePtr;
207}
208
209void RecordingEngine::finalize()
210{
211 qCDebug(qLcFFmpegEncoder) << "Media recording finalizing";
212
213 Q_ASSERT(m_state == State::FormatsInitializing || m_state == State::EncodersInitializing
214 || m_state == State::Encoding);
215
216 Q_ASSERT((m_state == State::FormatsInitializing) == !!m_formatsInitializer);
217
218 m_formatsInitializer.reset();
219
220 forEachEncoder(f: &disconnectEncoderFromSource);
221 if (m_state != State::Encoding)
222 forEachEncoder(f: &EncoderThread::startEncoding, args: false);
223
224 const bool shouldWriteTrailer = m_state == State::Encoding;
225 m_state = State::Finalizing;
226
227 EncodingFinalizer *finalizer = new EncodingFinalizer(*this, shouldWriteTrailer);
228 finalizer->start();
229}
230
231void RecordingEngine::setPaused(bool paused)
232{
233 forEachEncoder(f: &EncoderThread::setPaused, args&: paused);
234}
235
236void RecordingEngine::setAutoStop(bool autoStop)
237{
238 m_autoStop = autoStop;
239 forEachEncoder(f: &EncoderThread::setAutoStop, args&: autoStop);
240 handleSourceEndOfStream();
241}
242
243void RecordingEngine::setMetaData(const QMediaMetaData &metaData)
244{
245 m_metaData = metaData;
246}
247
248void RecordingEngine::newTimeStamp(qint64 time)
249{
250 QMutexLocker locker(&m_timeMutex);
251 if (time > m_timeRecorded) {
252 m_timeRecorded = time;
253 emit durationChanged(duration: time);
254 }
255}
256
257bool RecordingEngine::isEndOfSourceStreams() const
258{
259 return allOfEncoders(f: &EncoderThread::isEndOfSourceStream);
260}
261
262void RecordingEngine::handleSourceEndOfStream()
263{
264 if (m_autoStop && isEndOfSourceStreams())
265 emit autoStopped();
266}
267
268void RecordingEngine::handleEncoderInitialization()
269{
270 Q_ASSERT(m_state == State::EncodersInitializing || m_state == State::Finalizing);
271
272 if (m_state == State::Finalizing)
273 return; // outdated event, drop it
274
275 ++m_initializedEncodersCount;
276
277 Q_ASSERT(m_initializedEncodersCount <= encodersCount());
278
279 if (m_initializedEncodersCount < encodersCount())
280 return;
281
282 Q_ASSERT(allOfEncoders(&EncoderThread::isInitialized));
283
284 qCDebug(qLcFFmpegEncoder) << "Encoders initialized; writing header";
285
286 avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(metaData: m_metaData);
287
288 const int res = avformat_write_header(s: avFormatContext(), options: nullptr);
289 if (res < 0) {
290 qWarning() << "could not write header, error:" << res << AVError(res);
291 emit sessionError(code: QMediaRecorder::ResourceError,
292 description: QLatin1StringView("Cannot start writing the stream"));
293 return;
294 }
295
296 qCDebug(qLcFFmpegEncoder) << "Stream header is successfully written";
297
298 m_state = State::Encoding;
299 m_muxer->start();
300 forEachEncoder(f: &EncoderThread::startEncoding, args: true);
301}
302
303void RecordingEngine::stopAndDeleteThreads()
304{
305 m_audioEncoders.clear();
306 m_videoEncoders.clear();
307 m_muxer.reset();
308}
309
310template <typename F, typename... Args>
311void RecordingEngine::forEachEncoder(F &&f, Args &&...args)
312{
313 for (auto &audioEncoder : m_audioEncoders)
314 std::invoke(f, audioEncoder.get(), args...);
315 for (auto &videoEncoder : m_videoEncoders)
316 std::invoke(f, videoEncoder.get(), args...);
317}
318
319template <typename F>
320bool RecordingEngine::allOfEncoders(F &&f) const
321{
322 auto predicate = [&f](const auto &encoder) { return std::invoke(f, encoder.get()); };
323
324 return std::all_of(m_audioEncoders.cbegin(), m_audioEncoders.cend(), predicate)
325 && std::all_of(m_videoEncoders.cbegin(), m_videoEncoders.cend(), predicate);
326}
327} // namespace QFFmpeg
328
329QT_END_NAMESPACE
330
331#include "moc_qffmpegrecordingengine_p.cpp"
332

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp