1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstreamermediaencoder_p.h"
5#include "qgstreamerintegration_p.h"
6#include "qgstreamerformatinfo_p.h"
7#include "qgstpipeline_p.h"
8#include "qgstreamermessage_p.h"
9#include <private/qplatformcamera_p.h>
10#include "qaudiodevice.h"
11#include <private/qmediastoragelocation_p.h>
12
13#include <qdebug.h>
14#include <qeventloop.h>
15#include <qstandardpaths.h>
16#include <qmimetype.h>
17#include <qloggingcategory.h>
18
19#include <gst/gsttagsetter.h>
20#include <gst/gstversion.h>
21#include <gst/video/video.h>
22#include <gst/pbutils/encoding-profile.h>
23
24static Q_LOGGING_CATEGORY(qLcMediaEncoderGst, "qt.multimedia.encoder")
25
26QT_BEGIN_NAMESPACE
27
28QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent)
29 : QPlatformMediaRecorder(parent),
30 audioPauseControl(*this),
31 videoPauseControl(*this)
32{
33 signalDurationChangedTimer.setInterval(100);
34 signalDurationChangedTimer.callOnTimeout(args: &signalDurationChangedTimer,
35 args: [this](){ durationChanged(position: duration()); });
36}
37
38QGstreamerMediaEncoder::~QGstreamerMediaEncoder()
39{
40 if (!gstPipeline.isNull()) {
41 finalize();
42 gstPipeline.removeMessageFilter(filter: this);
43 gstPipeline.setStateSync(GST_STATE_NULL);
44 }
45}
46
47bool QGstreamerMediaEncoder::isLocationWritable(const QUrl &) const
48{
49 return true;
50}
51
52void QGstreamerMediaEncoder::handleSessionError(QMediaRecorder::Error code, const QString &description)
53{
54 error(error: code, errorString: description);
55 stop();
56}
57
58bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &message)
59{
60 if (message.isNull())
61 return false;
62 auto msg = message;
63
64// qCDebug(qLcMediaEncoderGst) << "received event from" << message.source().name() << Qt::hex << message.type();
65// if (message.type() == GST_MESSAGE_STATE_CHANGED) {
66// GstState oldState;
67// GstState newState;
68// GstState pending;
69// gst_message_parse_state_changed(gm, &oldState, &newState, &pending);
70// qCDebug(qLcMediaEncoderGst) << "received state change from" << message.source().name() << oldState << newState << pending;
71// }
72 if (msg.type() == GST_MESSAGE_ELEMENT) {
73 QGstStructure s = msg.structure();
74 qCDebug(qLcMediaEncoderGst) << "received element message from" << msg.source().name() << s.name();
75 if (s.name() == "GstBinForwarded")
76 msg = QGstreamerMessage(s);
77 if (msg.isNull())
78 return false;
79 }
80
81 if (msg.type() == GST_MESSAGE_EOS) {
82 qCDebug(qLcMediaEncoderGst) << "received EOS from" << msg.source().name();
83 finalize();
84 return false;
85 }
86
87 if (msg.type() == GST_MESSAGE_ERROR) {
88 GError *err;
89 gchar *debug;
90 gst_message_parse_error(message: msg.rawMessage(), gerror: &err, debug: &debug);
91 error(error: QMediaRecorder::ResourceError, errorString: QString::fromUtf8(utf8: err->message));
92 g_error_free(error: err);
93 g_free(mem: debug);
94 if (!m_finalizing)
95 stop();
96 finalize();
97 }
98
99 return false;
100}
101
102qint64 QGstreamerMediaEncoder::duration() const
103{
104 return std::max(a: audioPauseControl.duration, b: videoPauseControl.duration);
105}
106
107
108static GstEncodingContainerProfile *createContainerProfile(const QMediaEncoderSettings &settings)
109{
110 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
111
112 auto caps = formatInfo->formatCaps(f: settings.fileFormat());
113
114 GstEncodingContainerProfile *profile = (GstEncodingContainerProfile *)gst_encoding_container_profile_new(
115 name: "container_profile",
116 description: (gchar *)"custom container profile",
117 format: const_cast<GstCaps *>(caps.get()),
118 preset: nullptr); //preset
119 return profile;
120}
121
122static GstEncodingProfile *createVideoProfile(const QMediaEncoderSettings &settings)
123{
124 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
125
126 auto caps = formatInfo->videoCaps(f: settings.mediaFormat());
127 if (caps.isNull())
128 return nullptr;
129
130 GstEncodingVideoProfile *profile = gst_encoding_video_profile_new(
131 format: const_cast<GstCaps *>(caps.get()),
132 preset: nullptr,
133 restriction: nullptr, //restriction
134 presence: 0); //presence
135
136 gst_encoding_video_profile_set_pass(prof: profile, pass: 0);
137 gst_encoding_video_profile_set_variableframerate(prof: profile, TRUE);
138
139 return (GstEncodingProfile *)profile;
140}
141
142static GstEncodingProfile *createAudioProfile(const QMediaEncoderSettings &settings)
143{
144 auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo();
145
146 auto caps = formatInfo->audioCaps(f: settings.mediaFormat());
147 if (caps.isNull())
148 return nullptr;
149
150 GstEncodingProfile *profile = (GstEncodingProfile *)gst_encoding_audio_profile_new(
151 format: const_cast<GstCaps *>(caps.get()),
152 preset: nullptr, //preset
153 restriction: nullptr, //restriction
154 presence: 0); //presence
155
156 return profile;
157}
158
159
160static GstEncodingContainerProfile *createEncodingProfile(const QMediaEncoderSettings &settings)
161{
162 auto *containerProfile = createContainerProfile(settings);
163 if (!containerProfile) {
164 qWarning() << "QGstreamerMediaEncoder: failed to create container profile!";
165 return nullptr;
166 }
167
168 GstEncodingProfile *audioProfile = createAudioProfile(settings);
169 GstEncodingProfile *videoProfile = nullptr;
170 if (settings.videoCodec() != QMediaFormat::VideoCodec::Unspecified)
171 videoProfile = createVideoProfile(settings);
172// qDebug() << "audio profile" << (audioProfile ? gst_caps_to_string(gst_encoding_profile_get_format(audioProfile)) : "(null)");
173// qDebug() << "video profile" << (videoProfile ? gst_caps_to_string(gst_encoding_profile_get_format(videoProfile)) : "(null)");
174// qDebug() << "conta profile" << gst_caps_to_string(gst_encoding_profile_get_format((GstEncodingProfile *)containerProfile));
175
176 if (videoProfile) {
177 if (!gst_encoding_container_profile_add_profile(container: containerProfile, profile: videoProfile)) {
178 qWarning() << "QGstreamerMediaEncoder: failed to add video profile!";
179 gst_encoding_profile_unref(videoProfile);
180 }
181 }
182 if (audioProfile) {
183 if (!gst_encoding_container_profile_add_profile(container: containerProfile, profile: audioProfile)) {
184 qWarning() << "QGstreamerMediaEncoder: failed to add audio profile!";
185 gst_encoding_profile_unref(audioProfile);
186 }
187 }
188
189 return containerProfile;
190}
191
192void QGstreamerMediaEncoder::PauseControl::reset()
193{
194 pauseOffsetPts = 0;
195 pauseStartPts.reset();
196 duration = 0;
197 firstBufferPts.reset();
198}
199
200void QGstreamerMediaEncoder::PauseControl::installOn(QGstPad pad)
201{
202 pad.addProbe<&QGstreamerMediaEncoder::PauseControl::processBuffer>(instance: this, type: GST_PAD_PROBE_TYPE_BUFFER);
203}
204
205GstPadProbeReturn QGstreamerMediaEncoder::PauseControl::processBuffer(QGstPad, GstPadProbeInfo *info)
206{
207 auto buffer = GST_PAD_PROBE_INFO_BUFFER(info);
208 if (!buffer)
209 return GST_PAD_PROBE_OK;
210
211 buffer = gst_buffer_make_writable(buffer);
212
213 if (!buffer)
214 return GST_PAD_PROBE_OK;
215
216 GST_PAD_PROBE_INFO_DATA(info) = buffer;
217
218 if (!GST_BUFFER_PTS_IS_VALID(buffer))
219 return GST_PAD_PROBE_OK;
220
221 if (!firstBufferPts)
222 firstBufferPts = GST_BUFFER_PTS(buffer);
223
224 if (encoder.state() == QMediaRecorder::PausedState) {
225 if (!pauseStartPts)
226 pauseStartPts = GST_BUFFER_PTS(buffer);
227
228 return GST_PAD_PROBE_DROP;
229 }
230
231 if (pauseStartPts) {
232 pauseOffsetPts += GST_BUFFER_PTS(buffer) - *pauseStartPts;
233 pauseStartPts.reset();
234 }
235 GST_BUFFER_PTS(buffer) -= pauseOffsetPts;
236
237 duration = (GST_BUFFER_PTS(buffer) - *firstBufferPts) / GST_MSECOND;
238
239 return GST_PAD_PROBE_OK;
240}
241
242void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings)
243{
244 if (!m_session ||m_finalizing || state() != QMediaRecorder::StoppedState)
245 return;
246
247 const auto hasVideo = m_session->camera() && m_session->camera()->isActive();
248 const auto hasAudio = m_session->audioInput() != nullptr;
249
250 if (!hasVideo && !hasAudio) {
251 error(error: QMediaRecorder::ResourceError, errorString: QMediaRecorder::tr(s: "No camera or audio input"));
252 return;
253 }
254
255 const auto audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified;
256
257 auto primaryLocation = audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation;
258 auto container = settings.mimeType().preferredSuffix();
259 auto location = QMediaStorageLocation::generateFileName(requestedName: outputLocation().toLocalFile(), type: primaryLocation, extension: container);
260
261 QUrl actualSink = QUrl::fromLocalFile(localfile: QDir::currentPath()).resolved(relative: location);
262 qCDebug(qLcMediaEncoderGst) << "recording new video to" << actualSink;
263
264 Q_ASSERT(!actualSink.isEmpty());
265
266 gstEncoder = QGstElement("encodebin", "encodebin");
267 Q_ASSERT(gstEncoder);
268 auto *encodingProfile = createEncodingProfile(settings);
269 g_object_set (object: gstEncoder.object(), first_property_name: "profile", encodingProfile, nullptr);
270 gst_encoding_profile_unref(encodingProfile);
271
272 gstFileSink = QGstElement("filesink", "filesink");
273 Q_ASSERT(gstFileSink);
274 gstFileSink.set(property: "location", str: QFile::encodeName(fileName: actualSink.toLocalFile()).constData());
275 gstFileSink.set(property: "async", b: false);
276
277 QGstPad audioSink = {};
278 QGstPad videoSink = {};
279
280 audioPauseControl.reset();
281 videoPauseControl.reset();
282
283 if (hasAudio) {
284 audioSink = gstEncoder.getRequestPad(name: "audio_%u");
285 if (audioSink.isNull())
286 qWarning() << "Unsupported audio codec";
287 else
288 audioPauseControl.installOn(pad: audioSink);
289 }
290
291 if (hasVideo) {
292 videoSink = gstEncoder.getRequestPad(name: "video_%u");
293 if (videoSink.isNull())
294 qWarning() << "Unsupported video codec";
295 else
296 videoPauseControl.installOn(pad: videoSink);
297 }
298
299 gstPipeline.add(e1: gstEncoder, e2: gstFileSink);
300 gstEncoder.link(next: gstFileSink);
301 m_metaData.setMetaData(gstEncoder.bin());
302
303 m_session->linkEncoder(audioSink, videoSink);
304
305 gstEncoder.syncStateWithParent();
306 gstFileSink.syncStateWithParent();
307
308 signalDurationChangedTimer.start();
309 gstPipeline.dumpGraph(fileName: "recording");
310
311 durationChanged(position: 0);
312 stateChanged(state: QMediaRecorder::RecordingState);
313 actualLocationChanged(location: QUrl::fromLocalFile(localfile: location));
314}
315
316void QGstreamerMediaEncoder::pause()
317{
318 if (!m_session || m_finalizing || state() != QMediaRecorder::RecordingState)
319 return;
320 signalDurationChangedTimer.stop();
321 gstPipeline.dumpGraph(fileName: "before-pause");
322 stateChanged(state: QMediaRecorder::PausedState);
323}
324
325void QGstreamerMediaEncoder::resume()
326{
327 gstPipeline.dumpGraph(fileName: "before-resume");
328 if (!m_session || m_finalizing || state() != QMediaRecorder::PausedState)
329 return;
330 signalDurationChangedTimer.start();
331 stateChanged(state: QMediaRecorder::RecordingState);
332}
333
334void QGstreamerMediaEncoder::stop()
335{
336 if (!m_session || m_finalizing || state() == QMediaRecorder::StoppedState)
337 return;
338 qCDebug(qLcMediaEncoderGst) << "stop";
339 m_finalizing = true;
340 m_session->unlinkEncoder();
341 signalDurationChangedTimer.stop();
342
343 qCDebug(qLcMediaEncoderGst) << ">>>>>>>>>>>>> sending EOS";
344 gstEncoder.sendEos();
345}
346
347void QGstreamerMediaEncoder::finalize()
348{
349 if (!m_session || gstEncoder.isNull())
350 return;
351
352 qCDebug(qLcMediaEncoderGst) << "finalize";
353
354 gstPipeline.remove(element: gstEncoder);
355 gstPipeline.remove(element: gstFileSink);
356 gstEncoder.setStateSync(GST_STATE_NULL);
357 gstFileSink.setStateSync(GST_STATE_NULL);
358 gstFileSink = {};
359 gstEncoder = {};
360 m_finalizing = false;
361 stateChanged(state: QMediaRecorder::StoppedState);
362}
363
364void QGstreamerMediaEncoder::setMetaData(const QMediaMetaData &metaData)
365{
366 if (!m_session)
367 return;
368 m_metaData = static_cast<const QGstreamerMetaData &>(metaData);
369}
370
371QMediaMetaData QGstreamerMediaEncoder::metaData() const
372{
373 return m_metaData;
374}
375
376void QGstreamerMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *session)
377{
378 QGstreamerMediaCapture *captureSession = static_cast<QGstreamerMediaCapture *>(session);
379 if (m_session == captureSession)
380 return;
381
382 if (m_session) {
383 stop();
384 if (m_finalizing) {
385 QEventLoop loop;
386 loop.connect(asender: mediaRecorder(), SIGNAL(recorderStateChanged(RecorderState)), SLOT(quit()));
387 loop.exec();
388 }
389
390 gstPipeline.removeMessageFilter(filter: this);
391 gstPipeline = {};
392 }
393
394 m_session = captureSession;
395 if (!m_session)
396 return;
397
398 gstPipeline = captureSession->gstPipeline;
399 gstPipeline.set(property: "message-forward", b: true);
400 gstPipeline.installMessageFilter(filter: this);
401}
402
403QT_END_NAMESPACE
404

source code of qtmultimedia/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp