1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <mediacapture/qgstreamermediarecorder_p.h>
5#include <qgstreamerformatinfo_p.h>
6#include <common/qgstpipeline_p.h>
7#include <common/qgstreamermessage_p.h>
8#include <common/qgst_debug_p.h>
9#include <qgstreamerintegration_p.h>
10
11#include <QtMultimedia/private/qmediastoragelocation_p.h>
12#include <QtMultimedia/private/qplatformcamera_p.h>
13#include <QtMultimedia/qaudiodevice.h>
14
15#include <QtCore/qdebug.h>
16#include <QtCore/qeventloop.h>
17#include <QtCore/qstandardpaths.h>
18#include <QtCore/qloggingcategory.h>
19
20#include <gst/gsttagsetter.h>
21#include <gst/gstversion.h>
22#include <gst/video/video.h>
23#include <gst/pbutils/encoding-profile.h>
24
25static Q_LOGGING_CATEGORY(qLcMediaRecorder, "qt.multimedia.encoder")
26
27QT_BEGIN_NAMESPACE
28
29QGstreamerMediaRecorder::QGstreamerMediaRecorder(QMediaRecorder *parent)
30 : QPlatformMediaRecorder(parent), audioPauseControl(*this), videoPauseControl(*this)
31{
32 signalDurationChangedTimer.setInterval(100);
33 signalDurationChangedTimer.callOnTimeout(args: &signalDurationChangedTimer, args: [this]() {
34 durationChanged(position: duration());
35 });
36}
37
38QGstreamerMediaRecorder::~QGstreamerMediaRecorder()
39{
40 if (m_session)
41 finalize();
42}
43
44bool QGstreamerMediaRecorder::isLocationWritable(const QUrl &) const
45{
46 return true;
47}
48
49void QGstreamerMediaRecorder::handleSessionError(QMediaRecorder::Error code,
50 const QString &description)
51{
52 updateError(error: code, errorString: description);
53 stop();
54}
55
56void QGstreamerMediaRecorder::processBusMessage(const QGstreamerMessage &msg)
57{
58 constexpr bool traceStateChange = false;
59 constexpr bool traceAllEvents = false;
60
61 if constexpr (traceAllEvents)
62 qCDebug(qLcMediaRecorder) << "received event:" << msg;
63
64 switch (msg.type()) {
65 case GST_MESSAGE_ELEMENT: {
66 QGstStructureView s = msg.structure();
67 if (s.name() == "GstBinForwarded")
68 return processBusMessage(msg: s.getMessage());
69
70 qCDebug(qLcMediaRecorder) << "received element message from" << msg.source().name()
71 << s.name();
72 return;
73 }
74
75 case GST_MESSAGE_EOS: {
76 qCDebug(qLcMediaRecorder) << "received EOS from" << msg.source().name();
77 finalize();
78 return;
79 }
80
81 case GST_MESSAGE_ERROR: {
82 qCDebug(qLcMediaRecorder) << "received error:" << msg.source().name()
83 << QCompactGstMessageAdaptor(msg);
84
85 QUniqueGErrorHandle err;
86 QGString debug;
87 gst_message_parse_error(message: msg.message(), gerror: &err, debug: &debug);
88 updateError(error: QMediaRecorder::ResourceError, errorString: QString::fromUtf8(utf8: err.get()->message));
89 if (!m_finalizing)
90 stop();
91 finalize();
92 return;
93 }
94
95 case GST_MESSAGE_STATE_CHANGED: {
96 if constexpr (traceStateChange)
97 qCDebug(qLcMediaRecorder) << "received state change" << QCompactGstMessageAdaptor(msg);
98
99 return;
100 }
101
102 default:
103 return;
104 };
105}
106
107qint64 QGstreamerMediaRecorder::duration() const
108{
109 return std::max(a: audioPauseControl.duration, b: videoPauseControl.duration);
110}
111
112
113static GstEncodingContainerProfile *createContainerProfile(const QMediaEncoderSettings &settings)
114{
115 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
116
117 auto caps = formatInfo->formatCaps(f: settings.fileFormat());
118
119 GstEncodingContainerProfile *profile =
120 (GstEncodingContainerProfile *)gst_encoding_container_profile_new(
121 name: "container_profile", description: (gchar *)"custom container profile",
122 format: const_cast<GstCaps *>(caps.caps()),
123 preset: nullptr); // preset
124 return profile;
125}
126
127static GstEncodingProfile *createVideoProfile(const QMediaEncoderSettings &settings)
128{
129 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
130
131 QGstCaps caps = formatInfo->videoCaps(f: settings.mediaFormat());
132 if (!caps)
133 return nullptr;
134
135 QSize videoResolution = settings.videoResolution();
136 if (videoResolution.isValid())
137 caps.setResolution(videoResolution);
138
139 GstEncodingVideoProfile *profile =
140 gst_encoding_video_profile_new(format: const_cast<GstCaps *>(caps.caps()), preset: nullptr,
141 restriction: nullptr, // restriction
142 presence: 0); // presence
143
144 gst_encoding_video_profile_set_pass(prof: profile, pass: 0);
145 gst_encoding_video_profile_set_variableframerate(prof: profile, TRUE);
146
147 return (GstEncodingProfile *)profile;
148}
149
150static GstEncodingProfile *createAudioProfile(const QMediaEncoderSettings &settings)
151{
152 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
153
154 auto caps = formatInfo->audioCaps(f: settings.mediaFormat());
155 if (!caps)
156 return nullptr;
157
158 GstEncodingProfile *profile =
159 (GstEncodingProfile *)gst_encoding_audio_profile_new(format: const_cast<GstCaps *>(caps.caps()),
160 preset: nullptr, // preset
161 restriction: nullptr, // restriction
162 presence: 0); // presence
163
164 return profile;
165}
166
167
168static GstEncodingContainerProfile *createEncodingProfile(const QMediaEncoderSettings &settings)
169{
170 auto *containerProfile = createContainerProfile(settings);
171 if (!containerProfile) {
172 qWarning() << "QGstreamerMediaEncoder: failed to create container profile!";
173 return nullptr;
174 }
175
176 GstEncodingProfile *audioProfile = createAudioProfile(settings);
177 GstEncodingProfile *videoProfile = nullptr;
178 if (settings.videoCodec() != QMediaFormat::VideoCodec::Unspecified)
179 videoProfile = createVideoProfile(settings);
180// qDebug() << "audio profile" << (audioProfile ? gst_caps_to_string(gst_encoding_profile_get_format(audioProfile)) : "(null)");
181// qDebug() << "video profile" << (videoProfile ? gst_caps_to_string(gst_encoding_profile_get_format(videoProfile)) : "(null)");
182// qDebug() << "conta profile" << gst_caps_to_string(gst_encoding_profile_get_format((GstEncodingProfile *)containerProfile));
183
184 if (videoProfile) {
185 if (!gst_encoding_container_profile_add_profile(container: containerProfile, profile: videoProfile)) {
186 qWarning() << "QGstreamerMediaEncoder: failed to add video profile!";
187 gst_encoding_profile_unref(videoProfile);
188 }
189 }
190 if (audioProfile) {
191 if (!gst_encoding_container_profile_add_profile(container: containerProfile, profile: audioProfile)) {
192 qWarning() << "QGstreamerMediaEncoder: failed to add audio profile!";
193 gst_encoding_profile_unref(audioProfile);
194 }
195 }
196
197 return containerProfile;
198}
199
200void QGstreamerMediaRecorder::PauseControl::reset()
201{
202 pauseOffsetPts = 0;
203 pauseStartPts.reset();
204 duration = 0;
205 firstBufferPts.reset();
206}
207
208void QGstreamerMediaRecorder::PauseControl::installOn(QGstPad pad)
209{
210 pad.addProbe<&QGstreamerMediaRecorder::PauseControl::processBuffer>(instance: this,
211 type: GST_PAD_PROBE_TYPE_BUFFER);
212}
213
214GstPadProbeReturn QGstreamerMediaRecorder::PauseControl::processBuffer(QGstPad,
215 GstPadProbeInfo *info)
216{
217 auto buffer = GST_PAD_PROBE_INFO_BUFFER(info);
218 if (!buffer)
219 return GST_PAD_PROBE_OK;
220
221 buffer = gst_buffer_make_writable(buffer);
222
223 if (!buffer)
224 return GST_PAD_PROBE_OK;
225
226 GST_PAD_PROBE_INFO_DATA(info) = buffer;
227
228 if (!GST_BUFFER_PTS_IS_VALID(buffer))
229 return GST_PAD_PROBE_OK;
230
231 if (!firstBufferPts)
232 firstBufferPts = GST_BUFFER_PTS(buffer);
233
234 if (encoder.state() == QMediaRecorder::PausedState) {
235 if (!pauseStartPts)
236 pauseStartPts = GST_BUFFER_PTS(buffer);
237
238 return GST_PAD_PROBE_DROP;
239 }
240
241 if (pauseStartPts) {
242 pauseOffsetPts += GST_BUFFER_PTS(buffer) - *pauseStartPts;
243 pauseStartPts.reset();
244 }
245 GST_BUFFER_PTS(buffer) -= pauseOffsetPts;
246
247 duration = (GST_BUFFER_PTS(buffer) - *firstBufferPts) / GST_MSECOND;
248
249 return GST_PAD_PROBE_OK;
250}
251
252void QGstreamerMediaRecorder::record(QMediaEncoderSettings &settings)
253{
254 if (!m_session ||m_finalizing || state() != QMediaRecorder::StoppedState)
255 return;
256
257 const auto hasVideo = m_session->camera() && m_session->camera()->isActive();
258 const auto hasAudio = m_session->audioInput() != nullptr;
259
260 if (!hasVideo && !hasAudio) {
261 updateError(error: QMediaRecorder::ResourceError, errorString: QMediaRecorder::tr(s: "No camera or audio input"));
262 return;
263 }
264
265 const auto audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified;
266
267 auto primaryLocation = audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation;
268 auto container = settings.preferredSuffix();
269 auto location = QMediaStorageLocation::generateFileName(requestedName: outputLocation().toLocalFile(), type: primaryLocation, extension: container);
270
271 QUrl actualSink = QUrl::fromLocalFile(localfile: QDir::currentPath()).resolved(relative: location);
272 qCDebug(qLcMediaRecorder) << "recording new video to" << actualSink;
273
274 Q_ASSERT(!actualSink.isEmpty());
275
276 QGstBin gstEncodebin = QGstBin::createFromFactory(factory: "encodebin", name: "encodebin");
277 Q_ASSERT(gstEncodebin);
278 auto *encodingProfile = createEncodingProfile(settings);
279 g_object_set(object: gstEncodebin.object(), first_property_name: "profile", encodingProfile, nullptr);
280 gst_encoding_profile_unref(encodingProfile);
281
282 QGstElement gstFileSink = QGstElement::createFromFactory(factory: "filesink", name: "filesink");
283 Q_ASSERT(gstFileSink);
284 gstFileSink.set(property: "location", str: QFile::encodeName(fileName: actualSink.toLocalFile()).constData());
285
286 QGstPad audioSink = {};
287 QGstPad videoSink = {};
288
289 audioPauseControl.reset();
290 videoPauseControl.reset();
291
292 if (hasAudio) {
293 audioSink = gstEncodebin.getRequestPad(name: "audio_%u");
294 if (!audioSink)
295 qWarning() << "Unsupported audio codec";
296 else
297 audioPauseControl.installOn(pad: audioSink);
298 }
299
300 if (hasVideo) {
301 videoSink = gstEncodebin.getRequestPad(name: "video_%u");
302 if (!videoSink)
303 qWarning() << "Unsupported video codec";
304 else
305 videoPauseControl.installOn(pad: videoSink);
306 }
307
308 QGstreamerMediaCaptureSession::RecorderElements recorder{
309 .encodeBin: std::move(gstEncodebin),
310 .fileSink: std::move(gstFileSink),
311 .audioSink: std::move(audioSink),
312 .videoSink: std::move(videoSink),
313 };
314
315 m_session->linkAndStartEncoder(std::move(recorder), m_metaData);
316
317 signalDurationChangedTimer.start();
318
319 m_session->pipeline().dumpGraph(fileNamePrefix: "recording");
320
321 durationChanged(position: 0);
322 actualLocationChanged(location: QUrl::fromLocalFile(localfile: location));
323 stateChanged(state: QMediaRecorder::RecordingState);
324}
325
326void QGstreamerMediaRecorder::pause()
327{
328 if (!m_session || m_finalizing || state() != QMediaRecorder::RecordingState)
329 return;
330 signalDurationChangedTimer.stop();
331 durationChanged(position: duration());
332 m_session->pipeline().dumpGraph(fileNamePrefix: "before-pause");
333 stateChanged(state: QMediaRecorder::PausedState);
334}
335
336void QGstreamerMediaRecorder::resume()
337{
338 m_session->pipeline().dumpGraph(fileNamePrefix: "before-resume");
339 if (!m_session || m_finalizing || state() != QMediaRecorder::PausedState)
340 return;
341 signalDurationChangedTimer.start();
342 stateChanged(state: QMediaRecorder::RecordingState);
343}
344
345void QGstreamerMediaRecorder::stop()
346{
347 if (!m_session || m_finalizing || state() == QMediaRecorder::StoppedState)
348 return;
349 durationChanged(position: duration());
350 qCDebug(qLcMediaRecorder) << "stop";
351 m_finalizing = true;
352 m_session->unlinkRecorder();
353 signalDurationChangedTimer.stop();
354}
355
356void QGstreamerMediaRecorder::finalize()
357{
358 if (!m_session || !m_finalizing)
359 return;
360
361 qCDebug(qLcMediaRecorder) << "finalize";
362
363 m_session->finalizeRecorder();
364 m_finalizing = false;
365 stateChanged(state: QMediaRecorder::StoppedState);
366}
367
368void QGstreamerMediaRecorder::setMetaData(const QMediaMetaData &metaData)
369{
370 if (!m_session)
371 return;
372 m_metaData = metaData;
373}
374
375QMediaMetaData QGstreamerMediaRecorder::metaData() const
376{
377 return m_metaData;
378}
379
380void QGstreamerMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *session)
381{
382 QGstreamerMediaCaptureSession *captureSession =
383 static_cast<QGstreamerMediaCaptureSession *>(session);
384 if (m_session == captureSession)
385 return;
386
387 if (m_session) {
388 stop();
389 if (m_finalizing) {
390 QEventLoop loop;
391 QObject::connect(sender: mediaRecorder(), signal: &QMediaRecorder::recorderStateChanged, context: &loop,
392 slot: &QEventLoop::quit);
393 loop.exec();
394 }
395 }
396
397 m_session = captureSession;
398}
399
400QT_END_NAMESPACE
401

source code of qtmultimedia/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediarecorder.cpp