1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3#include "qffmpegrecordingengine_p.h"
4#include "qffmpegencodinginitializer_p.h"
5#include "qffmpegaudioencoder_p.h"
6#include "qffmpegaudioinput_p.h"
7#include "qffmpegrecordingengineutils_p.h"
8
9#include "private/qmultimediautils_p.h"
10#include "private/qplatformaudiobufferinput_p.h"
11#include "private/qplatformvideosource_p.h"
12#include "private/qplatformvideoframeinput_p.h"
13
14#include "qdebug.h"
15#include "qffmpegvideoencoder_p.h"
16#include "qffmpegmediametadata_p.h"
17#include "qffmpegmuxer_p.h"
18#include "qloggingcategory.h"
19
20QT_BEGIN_NAMESPACE
21
22static Q_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder");
23
24namespace QFFmpeg
25{
26
27RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings,
28 std::unique_ptr<EncodingFormatContext> context)
29 : m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this))
30{
31 Q_ASSERT(m_formatContext);
32 Q_ASSERT(m_formatContext->isAVIOOpen());
33}
34
35RecordingEngine::~RecordingEngine()
36{
37 Q_ASSERT(m_state == State::Finalizing);
38}
39
40void RecordingEngine::addAudioInput(QFFmpegAudioInput *input)
41{
42 Q_ASSERT(input);
43 Q_ASSERT(m_state == State::FormatsInitializing);
44
45 if (input->device.isNull()) {
46 emit streamInitializationError(code: QMediaRecorder::ResourceError,
47 description: QLatin1StringView("Audio device is null"));
48 return;
49 }
50
51 const QAudioFormat format = input->device.preferredFormat();
52
53 if (!format.isValid()) {
54 emit streamInitializationError(
55 code: QMediaRecorder::FormatError,
56 description: QLatin1StringView("Audio device has invalid preferred format"));
57 return;
58 }
59
60 AudioEncoder *audioEncoder = createAudioEncoder(format);
61 connectEncoderToSource(encoder: audioEncoder, source: input);
62
63 input->setRunning(true);
64}
65
66void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input,
67 const QAudioBuffer &firstBuffer)
68{
69 Q_ASSERT(input);
70 Q_ASSERT(m_state == State::FormatsInitializing);
71
72 const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat();
73
74 AudioEncoder *audioEncoder = createAudioEncoder(format);
75
76 // set the buffer before connecting to avoid potential races
77 if (firstBuffer.isValid())
78 audioEncoder->addBuffer(buffer: firstBuffer);
79
80 connectEncoderToSource(encoder: audioEncoder, source: input);
81}
82
83AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format)
84{
85 Q_ASSERT(format.isValid());
86
87 auto audioEncoder = new AudioEncoder(*this, format, m_settings);
88
89 m_audioEncoders.emplace_back(args&: audioEncoder);
90 connect(sender: audioEncoder, signal: &EncoderThread::endOfSourceStream, context: this,
91 slot: &RecordingEngine::handleSourceEndOfStream);
92 connect(sender: audioEncoder, signal: &EncoderThread::initialized, context: this,
93 slot: &RecordingEngine::handleEncoderInitialization, type: Qt::SingleShotConnection);
94 if (m_autoStop)
95 audioEncoder->setAutoStop(true);
96
97 return audioEncoder;
98}
99
100void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame)
101{
102 Q_ASSERT(m_state == State::FormatsInitializing);
103
104 QVideoFrameFormat frameFormat =
105 firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat();
106
107 Q_ASSERT(frameFormat.isValid());
108
109 if (firstFrame.isValid() && frameFormat.streamFrameRate() <= 0.f) {
110 const qint64 startTime = firstFrame.startTime();
111 const qint64 endTime = firstFrame.endTime();
112 if (startTime != -1 && endTime > startTime)
113 frameFormat.setStreamFrameRate(static_cast<qreal>(VideoFrameTimeBase)
114 / (endTime - startTime));
115 }
116
117 std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat()
118 ? AVPixelFormat(*source->ffmpegHWPixelFormat())
119 : std::optional<AVPixelFormat>{};
120
121 qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":"
122 << "pixelFormat=" << frameFormat.pixelFormat()
123 << "frameSize=" << frameFormat.frameSize()
124 << "frameRate=" << frameFormat.streamFrameRate()
125 << "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE);
126
127 auto videoEncoder = new VideoEncoder(*this, m_settings, frameFormat, hwPixelFormat);
128 m_videoEncoders.emplace_back(args&: videoEncoder);
129 if (m_autoStop)
130 videoEncoder->setAutoStop(true);
131
132 connect(sender: videoEncoder, signal: &EncoderThread::endOfSourceStream, context: this,
133 slot: &RecordingEngine::handleSourceEndOfStream);
134
135 connect(sender: videoEncoder, signal: &EncoderThread::initialized, context: this,
136 slot: &RecordingEngine::handleEncoderInitialization, type: Qt::SingleShotConnection);
137
138 // set the frame before connecting to avoid potential races
139 if (firstFrame.isValid())
140 videoEncoder->addFrame(frame: firstFrame);
141
142 connectEncoderToSource(encoder: videoEncoder, source);
143}
144
145bool RecordingEngine::startEncoders()
146{
147 Q_ASSERT(m_state == State::FormatsInitializing);
148 Q_ASSERT(m_formatsInitializer);
149 m_formatsInitializer.reset();
150
151 if (m_audioEncoders.empty() && m_videoEncoders.empty()) {
152 emit sessionError(code: QMediaRecorder::ResourceError,
153 description: QLatin1StringView("No valid stream found for encoding"));
154 return false;
155 }
156
157 m_state = State::EncodersInitializing;
158
159 forEachEncoder(f: [](EncoderThread *encoder) { //
160 encoder->start();
161 });
162
163 return true;
164}
165
166bool RecordingEngine::initialize(const std::vector<QAudioBufferSource *> &audioSources,
167 const std::vector<QPlatformVideoSource *> &videoSources)
168{
169 Q_ASSERT(m_state == State::None);
170
171 m_state = State::FormatsInitializing;
172 m_formatsInitializer = std::make_unique<EncodingInitializer>(args&: *this);
173 return m_formatsInitializer->start(audioSources, videoSources);
174}
175
176RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine,
177 bool writeTrailer)
178 : m_recordingEngine(recordingEngine), m_writeTrailer(writeTrailer)
179{
180 Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
181 connect(sender: this, signal: &QThread::finished, context: this, slot: &QObject::deleteLater);
182}
183
184void RecordingEngine::EncodingFinalizer::run()
185{
186 Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
187
188 m_recordingEngine.stopAndDeleteThreads();
189
190 if (m_writeTrailer) {
191 const int res = av_write_trailer(s: m_recordingEngine.avFormatContext());
192 if (res < 0) {
193 const auto errorDescription = err2str(errnum: res);
194 qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << errorDescription;
195 emit m_recordingEngine.sessionError(code: QMediaRecorder::FormatError,
196 description: QLatin1String("Cannot write trailer: ")
197 + errorDescription);
198 }
199 }
200 // else ffmpeg might crash
201
202 // close AVIO before emitting finalizationDone.
203 m_recordingEngine.m_formatContext->closeAVIO();
204
205 qCDebug(qLcFFmpegEncoder) << "Media recording finalized";
206 emit m_recordingEngine.finalizationDone();
207 auto recordingEnginePtr = &m_recordingEngine;
208 delete recordingEnginePtr;
209}
210
211void RecordingEngine::finalize()
212{
213 qCDebug(qLcFFmpegEncoder) << "Media recording finalizing";
214
215 Q_ASSERT(m_state == State::FormatsInitializing || m_state == State::EncodersInitializing
216 || m_state == State::Encoding);
217
218 Q_ASSERT((m_state == State::FormatsInitializing) == !!m_formatsInitializer);
219
220 m_formatsInitializer.reset();
221
222 forEachEncoder(f: &disconnectEncoderFromSource);
223 if (m_state != State::Encoding)
224 forEachEncoder(f: &EncoderThread::startEncoding, args: false);
225
226 const bool shouldWriteTrailer = m_state == State::Encoding;
227 m_state = State::Finalizing;
228
229 EncodingFinalizer *finalizer = new EncodingFinalizer(*this, shouldWriteTrailer);
230 finalizer->start();
231}
232
233void RecordingEngine::setPaused(bool paused)
234{
235 forEachEncoder(f: &EncoderThread::setPaused, args&: paused);
236}
237
238void RecordingEngine::setAutoStop(bool autoStop)
239{
240 m_autoStop = autoStop;
241 forEachEncoder(f: &EncoderThread::setAutoStop, args&: autoStop);
242 handleSourceEndOfStream();
243}
244
245void RecordingEngine::setMetaData(const QMediaMetaData &metaData)
246{
247 m_metaData = metaData;
248}
249
250void RecordingEngine::newTimeStamp(qint64 time)
251{
252 QMutexLocker locker(&m_timeMutex);
253 if (time > m_timeRecorded) {
254 m_timeRecorded = time;
255 emit durationChanged(duration: time);
256 }
257}
258
259bool RecordingEngine::isEndOfSourceStreams() const
260{
261 return allOfEncoders(f: &EncoderThread::isEndOfSourceStream);
262}
263
264void RecordingEngine::handleSourceEndOfStream()
265{
266 if (m_autoStop && isEndOfSourceStreams())
267 emit autoStopped();
268}
269
270void RecordingEngine::handleEncoderInitialization()
271{
272 Q_ASSERT(m_state == State::EncodersInitializing || m_state == State::Finalizing);
273
274 if (m_state == State::Finalizing)
275 return; // outdated event, drop it
276
277 ++m_initializedEncodersCount;
278
279 Q_ASSERT(m_initializedEncodersCount <= encodersCount());
280
281 if (m_initializedEncodersCount < encodersCount())
282 return;
283
284 Q_ASSERT(allOfEncoders(&EncoderThread::isInitialized));
285
286 qCDebug(qLcFFmpegEncoder) << "Encoders initialized; writing header";
287
288 avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(metaData: m_metaData);
289
290 const int res = avformat_write_header(s: avFormatContext(), options: nullptr);
291 if (res < 0) {
292 qWarning() << "could not write header, error:" << res << err2str(errnum: res);
293 emit sessionError(code: QMediaRecorder::ResourceError,
294 description: QLatin1StringView("Cannot start writing the stream"));
295 return;
296 }
297
298 qCDebug(qLcFFmpegEncoder) << "Stream header is successfully written";
299
300 m_state = State::Encoding;
301 m_muxer->start();
302 forEachEncoder(f: &EncoderThread::startEncoding, args: true);
303}
304
305void RecordingEngine::stopAndDeleteThreads()
306{
307 m_audioEncoders.clear();
308 m_videoEncoders.clear();
309 m_muxer.reset();
310}
311
312template <typename F, typename... Args>
313void RecordingEngine::forEachEncoder(F &&f, Args &&...args)
314{
315 for (auto &audioEncoder : m_audioEncoders)
316 std::invoke(f, audioEncoder.get(), args...);
317 for (auto &videoEncoder : m_videoEncoders)
318 std::invoke(f, videoEncoder.get(), args...);
319}
320
321template <typename F>
322bool RecordingEngine::allOfEncoders(F &&f) const
323{
324 auto predicate = [&f](const auto &encoder) { return std::invoke(f, encoder.get()); };
325
326 return std::all_of(m_audioEncoders.cbegin(), m_audioEncoders.cend(), predicate)
327 && std::all_of(m_videoEncoders.cbegin(), m_videoEncoders.cend(), predicate);
328}
329} // namespace QFFmpeg
330
331QT_END_NAMESPACE
332
333#include "moc_qffmpegrecordingengine_p.cpp"
334

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp