1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstreamerimagecapture_p.h"
5
6#include <QtMultimedia/qvideoframeformat.h>
7#include <QtMultimedia/private/qmediastoragelocation_p.h>
8#include <QtMultimedia/private/qplatformcamera_p.h>
9#include <QtMultimedia/private/qplatformimagecapture_p.h>
10#include <QtMultimedia/private/qvideoframe_p.h>
11#include <QtCore/qdebug.h>
12#include <QtCore/qdir.h>
13#include <QtCore/qstandardpaths.h>
14#include <QtCore/qcoreapplication.h>
15#include <QtCore/qloggingcategory.h>
16
17#include <common/qgstreamermetadata_p.h>
18#include <common/qgstvideobuffer_p.h>
19#include <common/qgstutils_p.h>
20
21#include <utility>
22
23QT_BEGIN_NAMESPACE
24
25namespace {
26Q_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture")
27
28struct ThreadPoolSingleton
29{
30 QObject m_context;
31 QMutex m_poolMutex;
32 QThreadPool *m_instance{};
33 bool m_appUnderDestruction = false;
34
35 QThreadPool *get(const QMutexLocker<QMutex> &)
36 {
37 if (m_instance)
38 return m_instance;
39 if (m_appUnderDestruction || !qApp)
40 return nullptr;
41
42 using namespace std::chrono;
43
44 m_instance = new QThreadPool;
45 m_instance->setMaxThreadCount(1); // 1 thread;
46 static constexpr auto expiryTimeout = minutes(5);
47 m_instance->setExpiryTimeout(round<milliseconds>(d: expiryTimeout).count());
48
49 QObject::connect(qApp, signal: &QCoreApplication::aboutToQuit, context: &m_context, slot: [&] {
50 // we need to make sure that thread-local QRhi is destroyed before the application to
51 // prevent QTBUG-124189
52 QMutexLocker guard(&m_poolMutex);
53 delete m_instance;
54 m_instance = {};
55 m_appUnderDestruction = true;
56 });
57
58 QObject::connect(qApp, signal: &QCoreApplication::destroyed, context: &m_context, slot: [&] {
59 m_appUnderDestruction = false;
60 });
61 return m_instance;
62 }
63
64 template <typename Functor>
65 QFuture<void> run(Functor &&f)
66 {
67 QMutexLocker guard(&m_poolMutex);
68 QThreadPool *pool = get(guard);
69 if (!pool)
70 return QFuture<void>{};
71
72 return QtConcurrent::run(pool, std::forward<Functor>(f));
73 }
74};
75
76ThreadPoolSingleton s_threadPoolSingleton;
77
78}; // namespace
79
80QMaybe<QPlatformImageCapture *> QGstreamerImageCapture::create(QImageCapture *parent)
81{
82 static const auto error = qGstErrorMessageIfElementsNotAvailable(
83 arg: "queue", args: "capsfilter", args: "videoconvert", args: "jpegenc", args: "jifmux", args: "fakesink");
84 if (error)
85 return *error;
86
87 return new QGstreamerImageCapture(parent);
88}
89
90QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent)
91 : QPlatformImageCapture(parent),
92 QGstreamerBufferProbe(ProbeBuffers),
93 bin{
94 QGstBin::create(name: "imageCaptureBin"),
95 },
96 queue{
97 QGstElement::createFromFactory(factory: "queue", name: "imageCaptureQueue"),
98 },
99 filter{
100 QGstElement::createFromFactory(factory: "capsfilter", name: "filter"),
101 },
102 videoConvert{
103 QGstElement::createFromFactory(factory: "videoconvert", name: "imageCaptureConvert"),
104 },
105 encoder{
106 QGstElement::createFromFactory(factory: "jpegenc", name: "jpegEncoder"),
107 },
108 muxer{
109 QGstElement::createFromFactory(factory: "jifmux", name: "jpegMuxer"),
110 },
111 sink{
112 QGstElement::createFromFactory(factory: "fakesink", name: "imageCaptureSink"),
113 }
114{
115 // configures the queue to be fast, lightweight and non blocking
116 queue.set(property: "leaky", i: 2 /*downstream*/);
117 queue.set(property: "silent", b: true);
118 queue.set(property: "max-size-buffers", i: int(1));
119 queue.set(property: "max-size-bytes", i: int(0));
120 queue.set(property: "max-size-time", i: uint64_t(0));
121
122 bin.add(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
123 qLinkGstElements(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
124 bin.addGhostPad(child: queue, name: "sink");
125
126 addProbeToPad(pad: queue.staticPad(name: "src").pad(), downstream: false);
127
128 sink.set(property: "signal-handoffs", b: true);
129 sink.set(property: "async", b: false);
130 m_handoffConnection = sink.connect(name: "handoff", G_CALLBACK(&saveImageFilter), userData: this);
131}
132
133QGstreamerImageCapture::~QGstreamerImageCapture()
134{
135 bin.setStateSync(state: GST_STATE_NULL);
136
137 // wait for pending futures
138 auto pendingFutures = [&] {
139 QMutexLocker guard(&m_mutex);
140 return std::move(m_pendingFutures);
141 }();
142
143 for (QFuture<void> &pendingImage : pendingFutures)
144 pendingImage.waitForFinished();
145}
146
147bool QGstreamerImageCapture::isReadyForCapture() const
148{
149 QMutexLocker guard(&m_mutex);
150 return m_session && !passImage && cameraActive;
151}
152
153int QGstreamerImageCapture::capture(const QString &fileName)
154{
155 using namespace Qt::Literals;
156 QString path = QMediaStorageLocation::generateFileName(
157 requestedName: fileName, type: QStandardPaths::PicturesLocation, extension: u"jpg"_s);
158 return doCapture(fileName: path);
159}
160
161int QGstreamerImageCapture::captureToBuffer()
162{
163 return doCapture(fileName: QString());
164}
165
166int QGstreamerImageCapture::doCapture(const QString &fileName)
167{
168 qCDebug(qLcImageCaptureGst) << "do capture";
169
170 {
171 QMutexLocker guard(&m_mutex);
172 if (!m_session) {
173 invokeDeferred(fn: [this] {
174 emit error(id: -1, error: QImageCapture::ResourceError,
175 errorString: QPlatformImageCapture::msgImageCaptureNotSet());
176 });
177
178 qCDebug(qLcImageCaptureGst) << "error 1";
179 return -1;
180 }
181 if (!m_session->camera()) {
182 invokeDeferred(fn: [this] {
183 emit error(id: -1, error: QImageCapture::ResourceError, errorString: tr(s: "No camera available."));
184 });
185
186 qCDebug(qLcImageCaptureGst) << "error 2";
187 return -1;
188 }
189 if (passImage) {
190 invokeDeferred(fn: [this] {
191 emit error(id: -1, error: QImageCapture::NotReadyError,
192 errorString: QPlatformImageCapture::msgCameraNotReady());
193 });
194
195 qCDebug(qLcImageCaptureGst) << "error 3";
196 return -1;
197 }
198 m_lastId++;
199
200 pendingImages.enqueue(t: { .id: m_lastId, .filename: fileName, .metaData: QMediaMetaData{} });
201 // let one image pass the pipeline
202 passImage = true;
203 }
204
205 emit readyForCaptureChanged(ready: false);
206 return m_lastId;
207}
208
209void QGstreamerImageCapture::setResolution(const QSize &resolution)
210{
211 QGstCaps padCaps = bin.staticPad(name: "sink").currentCaps();
212 if (padCaps.isNull()) {
213 qDebug() << "Camera not ready";
214 return;
215 }
216 QGstCaps caps = padCaps.copy();
217 if (caps.isNull())
218 return;
219
220 gst_caps_set_simple(caps: caps.caps(), field: "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT,
221 resolution.height(), nullptr);
222 filter.set(property: "caps", c: caps);
223}
224
225// HACK: gcc-10 and earlier reject [=,this] when building with c++17
226#if __cplusplus >= 202002L
227# define EQ_THIS_CAPTURE =, this
228#else
229# define EQ_THIS_CAPTURE =
230#endif
231
232bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer)
233{
234 QMutexLocker guard(&m_mutex);
235
236 if (!passImage)
237 return false;
238 qCDebug(qLcImageCaptureGst) << "probe buffer";
239
240 QGstBufferHandle bufferHandle{
241 buffer,
242 QGstBufferHandle::NeedsRef,
243 };
244
245 passImage = false;
246
247 bool ready = isReadyForCapture();
248 invokeDeferred(fn: [this, ready] {
249 emit readyForCaptureChanged(ready);
250 });
251
252 QGstCaps caps = bin.staticPad(name: "sink").currentCaps();
253 auto memoryFormat = caps.memoryFormat();
254
255 GstVideoInfo previewInfo;
256 QVideoFrameFormat fmt;
257 auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo();
258 if (optionalFormatAndVideoInfo)
259 std::tie(args&: fmt, args&: previewInfo) = std::move(*optionalFormatAndVideoInfo);
260
261 int futureId = futureIDAllocator += 1;
262
263 // ensure QVideoFrame::toImage is executed on a worker thread that is joined before the
264 // qApplication is destroyed
265 QFuture<void> future = s_threadPoolSingleton.run(f: [EQ_THIS_CAPTURE]() mutable {
266 QMutexLocker guard(&m_mutex);
267 auto scopeExit = qScopeGuard(f: [&] {
268 m_pendingFutures.remove(key: futureId);
269 });
270
271 if (!m_session) {
272 qDebug() << "QGstreamerImageCapture::probeBuffer: no session";
273 return;
274 }
275
276 auto *sink = m_session->gstreamerVideoSink();
277 auto gstBuffer = std::make_unique<QGstVideoBuffer>(args: std::move(bufferHandle), args&: previewInfo,
278 args&: sink, args&: fmt, args&: memoryFormat);
279
280 QVideoFrame frame = QVideoFramePrivate::createFrame(buffer: std::move(gstBuffer), format: fmt);
281 QImage img = frame.toImage();
282 if (img.isNull()) {
283 qDebug() << "received a null image";
284 return;
285 }
286
287 QMediaMetaData imageMetaData = metaData();
288 imageMetaData.insert(k: QMediaMetaData::Resolution, value: frame.size());
289 pendingImages.head().metaData = std::move(imageMetaData);
290 PendingImage pendingImage = pendingImages.head();
291
292 invokeDeferred(fn: [this, pendingImage = std::move(pendingImage), frame = std::move(frame),
293 img = std::move(img)]() mutable {
294 emit imageExposed(requestId: pendingImage.id);
295 qCDebug(qLcImageCaptureGst) << "Image available!";
296 emit imageAvailable(requestId: pendingImage.id, buffer: frame);
297 emit imageCaptured(requestId: pendingImage.id, preview: img);
298 emit imageMetadataAvailable(id: pendingImage.id, pendingImage.metaData);
299 });
300 });
301
302 if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable
303 return true;
304
305 m_pendingFutures.insert(key: futureId, value: future);
306
307 return true;
308}
309
310#undef EQ_THIS_CAPTURE
311
312void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session)
313{
314 QMutexLocker guard(&m_mutex);
315 QGstreamerMediaCaptureSession *captureSession = static_cast<QGstreamerMediaCaptureSession *>(session);
316 if (m_session == captureSession)
317 return;
318
319 bool readyForCapture = isReadyForCapture();
320 if (m_session) {
321 disconnect(sender: m_session, signal: nullptr, receiver: this, member: nullptr);
322 m_lastId = 0;
323 pendingImages.clear();
324 passImage = false;
325 cameraActive = false;
326 }
327
328 m_session = captureSession;
329 if (!m_session) {
330 if (readyForCapture)
331 emit readyForCaptureChanged(ready: false);
332 return;
333 }
334
335 connect(sender: m_session, signal: &QPlatformMediaCaptureSession::cameraChanged, context: this,
336 slot: &QGstreamerImageCapture::onCameraChanged);
337 onCameraChanged();
338}
339
340void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m)
341{
342 {
343 QMutexLocker guard(&m_mutex);
344 QPlatformImageCapture::setMetaData(m);
345 }
346
347 // ensure taginject injects this metaData
348 applyMetaDataToTagSetter(metadata: m, muxer);
349}
350
351void QGstreamerImageCapture::cameraActiveChanged(bool active)
352{
353 qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active;
354 if (cameraActive == active)
355 return;
356 cameraActive = active;
357 qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture();
358 emit readyForCaptureChanged(ready: isReadyForCapture());
359}
360
361void QGstreamerImageCapture::onCameraChanged()
362{
363 QMutexLocker guard(&m_mutex);
364 if (m_session->camera()) {
365 cameraActiveChanged(active: m_session->camera()->isActive());
366 connect(sender: m_session->camera(), signal: &QPlatformCamera::activeChanged, context: this,
367 slot: &QGstreamerImageCapture::cameraActiveChanged);
368 } else {
369 cameraActiveChanged(active: false);
370 }
371}
372
373gboolean QGstreamerImageCapture::saveImageFilter(GstElement *, GstBuffer *buffer, GstPad *,
374 QGstreamerImageCapture *capture)
375{
376 capture->saveBufferToImage(buffer);
377 return true;
378}
379
380void QGstreamerImageCapture::saveBufferToImage(GstBuffer *buffer)
381{
382 QMutexLocker guard(&m_mutex);
383 passImage = false;
384
385 if (pendingImages.isEmpty())
386 return;
387
388 PendingImage imageData = pendingImages.dequeue();
389 if (imageData.filename.isEmpty())
390 return;
391
392 int id = futureIDAllocator++;
393 QGstBufferHandle bufferHandle{
394 buffer,
395 QGstBufferHandle::NeedsRef,
396 };
397
398 QFuture<void> saveImageFuture = QtConcurrent::run(f: [this, imageData, bufferHandle,
399 id]() mutable {
400 auto cleanup = qScopeGuard(f: [&] {
401 QMutexLocker guard(&m_mutex);
402 m_pendingFutures.remove(key: id);
403 });
404
405 qCDebug(qLcImageCaptureGst) << "saving image as" << imageData.filename;
406
407 QFile f(imageData.filename);
408 if (!f.open(flags: QFile::WriteOnly)) {
409 qCDebug(qLcImageCaptureGst) << " could not open image file for writing";
410 return;
411 }
412
413 GstMapInfo info;
414 GstBuffer *buffer = bufferHandle.get();
415 if (gst_buffer_map(buffer, info: &info, flags: GST_MAP_READ)) {
416 f.write(data: reinterpret_cast<const char *>(info.data), len: info.size);
417 gst_buffer_unmap(buffer, info: &info);
418 }
419 f.close();
420
421 QMetaObject::invokeMethod(object: this, function: [this, imageData = std::move(imageData)]() mutable {
422 emit imageSaved(requestId: imageData.id, fileName: imageData.filename);
423 });
424 });
425
426 m_pendingFutures.insert(key: id, value: saveImageFuture);
427}
428
429QImageEncoderSettings QGstreamerImageCapture::imageSettings() const
430{
431 return m_settings;
432}
433
434void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings)
435{
436 if (m_settings != settings) {
437 QSize resolution = settings.resolution();
438 if (m_settings.resolution() != resolution && !resolution.isEmpty())
439 setResolution(resolution);
440
441 m_settings = settings;
442 }
443}
444
445QT_END_NAMESPACE
446
447#include "moc_qgstreamerimagecapture_p.cpp"
448

Provided by KDAB

Privacy Policy
Learn to use CMake with our Intro Training
Find out more

source code of qtmultimedia/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp