1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpegmediacapturesession_p.h"
5
6#include "private/qplatformaudioinput_p.h"
7#include "private/qplatformaudiooutput_p.h"
8#include "private/qplatformsurfacecapture_p.h"
9#include "private/qplatformaudiobufferinput_p.h"
10#include "private/qplatformvideoframeinput_p.h"
11#include "private/qplatformcamera_p.h"
12
13#include "qffmpegimagecapture_p.h"
14#include "qffmpegmediarecorder_p.h"
15#include "qvideosink.h"
16#include "qffmpegaudioinput_p.h"
17#include "qaudiosink.h"
18#include "qaudiobuffer.h"
19#include "qaudiooutput.h"
20
21#include <qloggingcategory.h>
22
23QT_BEGIN_NAMESPACE
24
25using namespace Qt::StringLiterals;
26
27static Q_LOGGING_CATEGORY(qLcFFmpegMediaCaptureSession, "qt.multimedia.ffmpeg.mediacapturesession")
28
29static int preferredAudioSinkBufferSize(const QFFmpegAudioInput &input)
30{
31 // Heuristic params to avoid jittering
32 // TODO: investigate the reason of jittering and probably reduce the factor
33 constexpr int BufferSizeFactor = 2;
34 constexpr int BufferSizeExceeding = 4096;
35
36 return input.bufferSize() * BufferSizeFactor + BufferSizeExceeding;
37}
38
39QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession()
40{
41 connect(sender: this, signal: &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, context: this,
42 slot: &QFFmpegMediaCaptureSession::updateVideoFrameConnection);
43}
44
45QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() = default;
46
47QPlatformCamera *QFFmpegMediaCaptureSession::camera()
48{
49 return m_camera;
50}
51
52void QFFmpegMediaCaptureSession::setCamera(QPlatformCamera *camera)
53{
54 if (setVideoSource(source&: m_camera, newSource: camera))
55 emit cameraChanged();
56}
57
58QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::screenCapture()
59{
60 return m_screenCapture;
61}
62
63void QFFmpegMediaCaptureSession::setScreenCapture(QPlatformSurfaceCapture *screenCapture)
64{
65 if (setVideoSource(source&: m_screenCapture, newSource: screenCapture))
66 emit screenCaptureChanged();
67}
68
69QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::windowCapture()
70{
71 return m_windowCapture;
72}
73
74void QFFmpegMediaCaptureSession::setWindowCapture(QPlatformSurfaceCapture *windowCapture)
75{
76 if (setVideoSource(source&: m_windowCapture, newSource: windowCapture))
77 emit windowCaptureChanged();
78}
79
80QPlatformVideoFrameInput *QFFmpegMediaCaptureSession::videoFrameInput()
81{
82 return m_videoFrameInput;
83}
84
85void QFFmpegMediaCaptureSession::setVideoFrameInput(QPlatformVideoFrameInput *input)
86{
87 if (setVideoSource(source&: m_videoFrameInput, newSource: input))
88 emit videoFrameInputChanged();
89}
90
91QPlatformImageCapture *QFFmpegMediaCaptureSession::imageCapture()
92{
93 return m_imageCapture;
94}
95
96void QFFmpegMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture)
97{
98 if (m_imageCapture == imageCapture)
99 return;
100
101 if (m_imageCapture)
102 m_imageCapture->setCaptureSession(nullptr);
103
104 m_imageCapture = static_cast<QFFmpegImageCapture *>(imageCapture);
105
106 if (m_imageCapture)
107 m_imageCapture->setCaptureSession(this);
108
109 emit imageCaptureChanged();
110}
111
112void QFFmpegMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *recorder)
113{
114 auto *r = static_cast<QFFmpegMediaRecorder *>(recorder);
115 if (m_mediaRecorder == r)
116 return;
117
118 if (m_mediaRecorder)
119 m_mediaRecorder->setCaptureSession(nullptr);
120 m_mediaRecorder = r;
121 if (m_mediaRecorder)
122 m_mediaRecorder->setCaptureSession(this);
123
124 emit encoderChanged();
125}
126
127QPlatformMediaRecorder *QFFmpegMediaCaptureSession::mediaRecorder()
128{
129 return m_mediaRecorder;
130}
131
132void QFFmpegMediaCaptureSession::setAudioInput(QPlatformAudioInput *input)
133{
134 qCDebug(qLcFFmpegMediaCaptureSession)
135 << "set audio input:" << (input ? input->device.description() : u"null"_s);
136
137 auto ffmpegAudioInput = dynamic_cast<QFFmpegAudioInput *>(input);
138 Q_ASSERT(!!input == !!ffmpegAudioInput);
139
140 if (m_audioInput == ffmpegAudioInput)
141 return;
142
143 if (m_audioInput)
144 m_audioInput->q->disconnect(receiver: this);
145
146 m_audioInput = ffmpegAudioInput;
147 if (m_audioInput)
148 // TODO: implement the signal in QPlatformAudioInput and connect to it, QTBUG-112294
149 connect(sender: m_audioInput->q, signal: &QAudioInput::deviceChanged, context: this,
150 slot: &QFFmpegMediaCaptureSession::updateAudioSink);
151
152 updateAudioSink();
153}
154
155void QFFmpegMediaCaptureSession::setAudioBufferInput(QPlatformAudioBufferInput *input)
156{
157 // TODO: implement binding to audio sink like setAudioInput does
158 m_audioBufferInput = input;
159}
160
161void QFFmpegMediaCaptureSession::updateAudioSink()
162{
163 if (m_audioSink) {
164 m_audioSink->reset();
165 m_audioSink.reset();
166 }
167
168 if (!m_audioInput || !m_audioOutput)
169 return;
170
171 auto format = m_audioInput->device.preferredFormat();
172
173 if (!m_audioOutput->device.isFormatSupported(format))
174 qWarning() << "Audio source format" << format << "is not compatible with the audio output";
175
176 m_audioSink = std::make_unique<QAudioSink>(args&: m_audioOutput->device, args&: format);
177
178 m_audioBufferSize = preferredAudioSinkBufferSize(input: *m_audioInput);
179 m_audioSink->setBufferSize(m_audioBufferSize);
180
181 qCDebug(qLcFFmpegMediaCaptureSession)
182 << "Create audiosink, format:" << format << "bufferSize:" << m_audioSink->bufferSize()
183 << "output device:" << m_audioOutput->device.description();
184
185 m_audioIODevice = m_audioSink->start();
186 if (m_audioIODevice) {
187 auto writeToDevice = [this](const QAudioBuffer &buffer) {
188 if (m_audioBufferSize < preferredAudioSinkBufferSize(input: *m_audioInput)) {
189 qCDebug(qLcFFmpegMediaCaptureSession)
190 << "Recreate audiosink due to small buffer size:" << m_audioBufferSize;
191
192 updateAudioSink();
193 }
194
195 const auto written =
196 m_audioIODevice->write(data: buffer.data<const char>(), len: buffer.byteCount());
197
198 if (written < buffer.byteCount())
199 qCWarning(qLcFFmpegMediaCaptureSession)
200 << "Not all bytes written:" << written << "vs" << buffer.byteCount();
201 };
202 connect(sender: m_audioInput, signal: &QFFmpegAudioInput::newAudioBuffer, context: m_audioSink.get(), slot&: writeToDevice);
203 } else {
204 qWarning() << "Failed to start audiosink push mode";
205 }
206
207 updateVolume();
208}
209
210void QFFmpegMediaCaptureSession::updateVolume()
211{
212 if (m_audioSink)
213 m_audioSink->setVolume(m_audioOutput->muted ? 0.f : m_audioOutput->volume);
214}
215
216QPlatformAudioInput *QFFmpegMediaCaptureSession::audioInput() const
217{
218 return m_audioInput;
219}
220
221void QFFmpegMediaCaptureSession::setVideoPreview(QVideoSink *sink)
222{
223 if (std::exchange(obj&: m_videoSink, new_val&: sink) == sink)
224 return;
225
226 updateVideoFrameConnection();
227}
228
229void QFFmpegMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output)
230{
231 qCDebug(qLcFFmpegMediaCaptureSession)
232 << "set audio output:" << (output ? output->device.description() : u"null"_s);
233
234 if (m_audioOutput == output)
235 return;
236
237 if (m_audioOutput)
238 m_audioOutput->q->disconnect(receiver: this);
239
240 m_audioOutput = output;
241 if (m_audioOutput) {
242 // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
243 connect(sender: m_audioOutput->q, signal: &QAudioOutput::deviceChanged, context: this,
244 slot: &QFFmpegMediaCaptureSession::updateAudioSink);
245 connect(sender: m_audioOutput->q, signal: &QAudioOutput::volumeChanged, context: this,
246 slot: &QFFmpegMediaCaptureSession::updateVolume);
247 connect(sender: m_audioOutput->q, signal: &QAudioOutput::mutedChanged, context: this,
248 slot: &QFFmpegMediaCaptureSession::updateVolume);
249 }
250
251 updateAudioSink();
252}
253
254void QFFmpegMediaCaptureSession::updateVideoFrameConnection()
255{
256 disconnect(m_videoFrameConnection);
257
258 if (m_primaryActiveVideoSource && m_videoSink) {
259 // deliver frames directly to video sink;
260 // AutoConnection type might be a pessimization due to an extra queuing
261 // TODO: investigate and integrate direct connection
262 m_videoFrameConnection =
263 connect(sender: m_primaryActiveVideoSource, signal: &QPlatformVideoSource::newVideoFrame,
264 context: m_videoSink, slot: &QVideoSink::setVideoFrame);
265 }
266}
267
268void QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource()
269{
270 auto sources = activeVideoSources();
271 auto source = sources.empty() ? nullptr : sources.front();
272 if (std::exchange(obj&: m_primaryActiveVideoSource, new_val&: source) != source)
273 emit primaryActiveVideoSourceChanged();
274}
275
276template<typename VideoSource>
277bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source,
278 VideoSource *newSource)
279{
280 if (source == newSource)
281 return false;
282
283 if (auto prevSource = std::exchange(source, newSource)) {
284 prevSource->setCaptureSession(nullptr);
285 prevSource->disconnect(this);
286 }
287
288 if (source) {
289 source->setCaptureSession(this);
290 connect(source, &QPlatformVideoSource::activeChanged, this,
291 &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource);
292 connect(source, &QObject::destroyed, this,
293 &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource, Qt::QueuedConnection);
294 }
295
296 updatePrimaryActiveVideoSource();
297
298 return true;
299}
300
301QPlatformVideoSource *QFFmpegMediaCaptureSession::primaryActiveVideoSource()
302{
303 return m_primaryActiveVideoSource;
304}
305
306std::vector<QAudioBufferSource *> QFFmpegMediaCaptureSession::activeAudioInputs() const
307{
308 std::vector<QAudioBufferSource *> result;
309 if (m_audioInput)
310 result.push_back(x: m_audioInput);
311
312 if (m_audioBufferInput)
313 result.push_back(x: m_audioBufferInput);
314
315 return result;
316}
317
318QT_END_NAMESPACE
319
320#include "moc_qffmpegmediacapturesession_p.cpp"
321

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp