1 | // Copyright (C) 2016 The Qt Company Ltd. |
---|---|
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | |
4 | #include "qffmpegmediacapturesession_p.h" |
5 | |
6 | #include "private/qplatformaudioinput_p.h" |
7 | #include "private/qplatformaudiooutput_p.h" |
8 | #include "private/qplatformsurfacecapture_p.h" |
9 | #include "private/qplatformaudiobufferinput_p.h" |
10 | #include "private/qplatformvideoframeinput_p.h" |
11 | #include "private/qplatformcamera_p.h" |
12 | |
13 | #include "qffmpegimagecapture_p.h" |
14 | #include "qffmpegmediarecorder_p.h" |
15 | #include "qvideosink.h" |
16 | #include "qffmpegaudioinput_p.h" |
17 | #include "qaudiosink.h" |
18 | #include "qaudiobuffer.h" |
19 | #include "qaudiooutput.h" |
20 | |
21 | #include <qloggingcategory.h> |
22 | |
23 | QT_BEGIN_NAMESPACE |
24 | |
25 | using namespace Qt::StringLiterals; |
26 | |
27 | static Q_LOGGING_CATEGORY(qLcFFmpegMediaCaptureSession, "qt.multimedia.ffmpeg.mediacapturesession") |
28 | |
29 | static int preferredAudioSinkBufferSize(const QFFmpegAudioInput &input) |
30 | { |
31 | // Heuristic params to avoid jittering |
32 | // TODO: investigate the reason of jittering and probably reduce the factor |
33 | constexpr int BufferSizeFactor = 2; |
34 | constexpr int BufferSizeExceeding = 4096; |
35 | |
36 | return input.bufferSize() * BufferSizeFactor + BufferSizeExceeding; |
37 | } |
38 | |
39 | QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession() |
40 | { |
41 | connect(sender: this, signal: &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, context: this, |
42 | slot: &QFFmpegMediaCaptureSession::updateVideoFrameConnection); |
43 | } |
44 | |
45 | QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() = default; |
46 | |
47 | QPlatformCamera *QFFmpegMediaCaptureSession::camera() |
48 | { |
49 | return m_camera; |
50 | } |
51 | |
52 | void QFFmpegMediaCaptureSession::setCamera(QPlatformCamera *camera) |
53 | { |
54 | if (setVideoSource(source&: m_camera, newSource: camera)) |
55 | emit cameraChanged(); |
56 | } |
57 | |
58 | QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::screenCapture() |
59 | { |
60 | return m_screenCapture; |
61 | } |
62 | |
63 | void QFFmpegMediaCaptureSession::setScreenCapture(QPlatformSurfaceCapture *screenCapture) |
64 | { |
65 | if (setVideoSource(source&: m_screenCapture, newSource: screenCapture)) |
66 | emit screenCaptureChanged(); |
67 | } |
68 | |
69 | QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::windowCapture() |
70 | { |
71 | return m_windowCapture; |
72 | } |
73 | |
74 | void QFFmpegMediaCaptureSession::setWindowCapture(QPlatformSurfaceCapture *windowCapture) |
75 | { |
76 | if (setVideoSource(source&: m_windowCapture, newSource: windowCapture)) |
77 | emit windowCaptureChanged(); |
78 | } |
79 | |
80 | QPlatformVideoFrameInput *QFFmpegMediaCaptureSession::videoFrameInput() |
81 | { |
82 | return m_videoFrameInput; |
83 | } |
84 | |
85 | void QFFmpegMediaCaptureSession::setVideoFrameInput(QPlatformVideoFrameInput *input) |
86 | { |
87 | if (setVideoSource(source&: m_videoFrameInput, newSource: input)) |
88 | emit videoFrameInputChanged(); |
89 | } |
90 | |
91 | QPlatformImageCapture *QFFmpegMediaCaptureSession::imageCapture() |
92 | { |
93 | return m_imageCapture; |
94 | } |
95 | |
96 | void QFFmpegMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture) |
97 | { |
98 | if (m_imageCapture == imageCapture) |
99 | return; |
100 | |
101 | if (m_imageCapture) |
102 | m_imageCapture->setCaptureSession(nullptr); |
103 | |
104 | m_imageCapture = static_cast<QFFmpegImageCapture *>(imageCapture); |
105 | |
106 | if (m_imageCapture) |
107 | m_imageCapture->setCaptureSession(this); |
108 | |
109 | emit imageCaptureChanged(); |
110 | } |
111 | |
112 | void QFFmpegMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *recorder) |
113 | { |
114 | auto *r = static_cast<QFFmpegMediaRecorder *>(recorder); |
115 | if (m_mediaRecorder == r) |
116 | return; |
117 | |
118 | if (m_mediaRecorder) |
119 | m_mediaRecorder->setCaptureSession(nullptr); |
120 | m_mediaRecorder = r; |
121 | if (m_mediaRecorder) |
122 | m_mediaRecorder->setCaptureSession(this); |
123 | |
124 | emit encoderChanged(); |
125 | } |
126 | |
127 | QPlatformMediaRecorder *QFFmpegMediaCaptureSession::mediaRecorder() |
128 | { |
129 | return m_mediaRecorder; |
130 | } |
131 | |
132 | void QFFmpegMediaCaptureSession::setAudioInput(QPlatformAudioInput *input) |
133 | { |
134 | qCDebug(qLcFFmpegMediaCaptureSession) |
135 | << "set audio input:"<< (input ? input->device.description() : u "null"_s); |
136 | |
137 | auto ffmpegAudioInput = dynamic_cast<QFFmpegAudioInput *>(input); |
138 | Q_ASSERT(!!input == !!ffmpegAudioInput); |
139 | |
140 | if (m_audioInput == ffmpegAudioInput) |
141 | return; |
142 | |
143 | if (m_audioInput) |
144 | m_audioInput->q->disconnect(receiver: this); |
145 | |
146 | m_audioInput = ffmpegAudioInput; |
147 | if (m_audioInput) |
148 | // TODO: implement the signal in QPlatformAudioInput and connect to it, QTBUG-112294 |
149 | connect(sender: m_audioInput->q, signal: &QAudioInput::deviceChanged, context: this, |
150 | slot: &QFFmpegMediaCaptureSession::updateAudioSink); |
151 | |
152 | updateAudioSink(); |
153 | } |
154 | |
155 | void QFFmpegMediaCaptureSession::setAudioBufferInput(QPlatformAudioBufferInput *input) |
156 | { |
157 | // TODO: implement binding to audio sink like setAudioInput does |
158 | m_audioBufferInput = input; |
159 | } |
160 | |
161 | void QFFmpegMediaCaptureSession::updateAudioSink() |
162 | { |
163 | if (m_audioSink) { |
164 | m_audioSink->reset(); |
165 | m_audioSink.reset(); |
166 | } |
167 | |
168 | if (!m_audioInput || !m_audioOutput) |
169 | return; |
170 | |
171 | auto format = m_audioInput->device.preferredFormat(); |
172 | |
173 | if (!m_audioOutput->device.isFormatSupported(format)) |
174 | qWarning() << "Audio source format"<< format << "is not compatible with the audio output"; |
175 | |
176 | m_audioSink = std::make_unique<QAudioSink>(args&: m_audioOutput->device, args&: format); |
177 | |
178 | m_audioBufferSize = preferredAudioSinkBufferSize(input: *m_audioInput); |
179 | m_audioSink->setBufferSize(m_audioBufferSize); |
180 | |
181 | qCDebug(qLcFFmpegMediaCaptureSession) |
182 | << "Create audiosink, format:"<< format << "bufferSize:"<< m_audioSink->bufferSize() |
183 | << "output device:"<< m_audioOutput->device.description(); |
184 | |
185 | m_audioIODevice = m_audioSink->start(); |
186 | if (m_audioIODevice) { |
187 | auto writeToDevice = [this](const QAudioBuffer &buffer) { |
188 | if (m_audioBufferSize < preferredAudioSinkBufferSize(input: *m_audioInput)) { |
189 | qCDebug(qLcFFmpegMediaCaptureSession) |
190 | << "Recreate audiosink due to small buffer size:"<< m_audioBufferSize; |
191 | |
192 | updateAudioSink(); |
193 | } |
194 | |
195 | const auto written = |
196 | m_audioIODevice->write(data: buffer.data<const char>(), len: buffer.byteCount()); |
197 | |
198 | if (written < buffer.byteCount()) |
199 | qCWarning(qLcFFmpegMediaCaptureSession) |
200 | << "Not all bytes written:"<< written << "vs"<< buffer.byteCount(); |
201 | }; |
202 | connect(sender: m_audioInput, signal: &QFFmpegAudioInput::newAudioBuffer, context: m_audioSink.get(), slot&: writeToDevice); |
203 | } else { |
204 | qWarning() << "Failed to start audiosink push mode"; |
205 | } |
206 | |
207 | updateVolume(); |
208 | } |
209 | |
210 | void QFFmpegMediaCaptureSession::updateVolume() |
211 | { |
212 | if (m_audioSink) |
213 | m_audioSink->setVolume(m_audioOutput->muted ? 0.f : m_audioOutput->volume); |
214 | } |
215 | |
216 | QPlatformAudioInput *QFFmpegMediaCaptureSession::audioInput() const |
217 | { |
218 | return m_audioInput; |
219 | } |
220 | |
221 | void QFFmpegMediaCaptureSession::setVideoPreview(QVideoSink *sink) |
222 | { |
223 | if (std::exchange(obj&: m_videoSink, new_val&: sink) == sink) |
224 | return; |
225 | |
226 | updateVideoFrameConnection(); |
227 | } |
228 | |
229 | void QFFmpegMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output) |
230 | { |
231 | qCDebug(qLcFFmpegMediaCaptureSession) |
232 | << "set audio output:"<< (output ? output->device.description() : u "null"_s); |
233 | |
234 | if (m_audioOutput == output) |
235 | return; |
236 | |
237 | if (m_audioOutput) |
238 | m_audioOutput->q->disconnect(receiver: this); |
239 | |
240 | m_audioOutput = output; |
241 | if (m_audioOutput) { |
242 | // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294 |
243 | connect(sender: m_audioOutput->q, signal: &QAudioOutput::deviceChanged, context: this, |
244 | slot: &QFFmpegMediaCaptureSession::updateAudioSink); |
245 | connect(sender: m_audioOutput->q, signal: &QAudioOutput::volumeChanged, context: this, |
246 | slot: &QFFmpegMediaCaptureSession::updateVolume); |
247 | connect(sender: m_audioOutput->q, signal: &QAudioOutput::mutedChanged, context: this, |
248 | slot: &QFFmpegMediaCaptureSession::updateVolume); |
249 | } |
250 | |
251 | updateAudioSink(); |
252 | } |
253 | |
254 | void QFFmpegMediaCaptureSession::updateVideoFrameConnection() |
255 | { |
256 | disconnect(m_videoFrameConnection); |
257 | |
258 | if (m_primaryActiveVideoSource && m_videoSink) { |
259 | // deliver frames directly to video sink; |
260 | // AutoConnection type might be a pessimization due to an extra queuing |
261 | // TODO: investigate and integrate direct connection |
262 | m_videoFrameConnection = |
263 | connect(sender: m_primaryActiveVideoSource, signal: &QPlatformVideoSource::newVideoFrame, |
264 | context: m_videoSink, slot: &QVideoSink::setVideoFrame); |
265 | } |
266 | } |
267 | |
268 | void QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource() |
269 | { |
270 | auto sources = activeVideoSources(); |
271 | auto source = sources.empty() ? nullptr : sources.front(); |
272 | if (std::exchange(obj&: m_primaryActiveVideoSource, new_val&: source) != source) |
273 | emit primaryActiveVideoSourceChanged(); |
274 | } |
275 | |
276 | template<typename VideoSource> |
277 | bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source, |
278 | VideoSource *newSource) |
279 | { |
280 | if (source == newSource) |
281 | return false; |
282 | |
283 | if (auto prevSource = std::exchange(source, newSource)) { |
284 | prevSource->setCaptureSession(nullptr); |
285 | prevSource->disconnect(this); |
286 | } |
287 | |
288 | if (source) { |
289 | source->setCaptureSession(this); |
290 | connect(source, &QPlatformVideoSource::activeChanged, this, |
291 | &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource); |
292 | connect(source, &QObject::destroyed, this, |
293 | &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource, Qt::QueuedConnection); |
294 | } |
295 | |
296 | updatePrimaryActiveVideoSource(); |
297 | |
298 | return true; |
299 | } |
300 | |
301 | QPlatformVideoSource *QFFmpegMediaCaptureSession::primaryActiveVideoSource() |
302 | { |
303 | return m_primaryActiveVideoSource; |
304 | } |
305 | |
306 | std::vector<QAudioBufferSource *> QFFmpegMediaCaptureSession::activeAudioInputs() const |
307 | { |
308 | std::vector<QAudioBufferSource *> result; |
309 | if (m_audioInput) |
310 | result.push_back(x: m_audioInput); |
311 | |
312 | if (m_audioBufferInput) |
313 | result.push_back(x: m_audioBufferInput); |
314 | |
315 | return result; |
316 | } |
317 | |
318 | QT_END_NAMESPACE |
319 | |
320 | #include "moc_qffmpegmediacapturesession_p.cpp" |
321 |
Definitions
- qLcFFmpegMediaCaptureSession
- preferredAudioSinkBufferSize
- QFFmpegMediaCaptureSession
- ~QFFmpegMediaCaptureSession
- camera
- setCamera
- screenCapture
- setScreenCapture
- windowCapture
- setWindowCapture
- videoFrameInput
- setVideoFrameInput
- imageCapture
- setImageCapture
- setMediaRecorder
- mediaRecorder
- setAudioInput
- setAudioBufferInput
- updateAudioSink
- updateVolume
- audioInput
- setVideoPreview
- setAudioOutput
- updateVideoFrameConnection
- updatePrimaryActiveVideoSource
- setVideoSource
- primaryActiveVideoSource
Start learning QML with our Intro Training
Find out more