1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstreamerimagecapture_p.h"
5
6#include <QtMultimedia/qvideoframeformat.h>
7#include <QtMultimedia/private/qmediastoragelocation_p.h>
8#include <QtMultimedia/private/qplatformcamera_p.h>
9#include <QtMultimedia/private/qplatformimagecapture_p.h>
10#include <QtMultimedia/private/qvideoframe_p.h>
11#include <QtGui/qguiapplication.h>
12#include <QtCore/qdebug.h>
13#include <QtCore/qdir.h>
14#include <QtCore/qstandardpaths.h>
15#include <QtCore/qcoreapplication.h>
16#include <QtCore/qloggingcategory.h>
17
18#include <common/qgstreamermetadata_p.h>
19#include <common/qgstvideobuffer_p.h>
20#include <common/qgstutils_p.h>
21
22#include <utility>
23
24QT_BEGIN_NAMESPACE
25
26namespace {
27
28Q_STATIC_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture")
29
30struct ThreadPoolSingleton
31{
32 QObject m_context;
33 QMutex m_poolMutex;
34 QThreadPool *m_instance{};
35 bool m_appUnderDestruction = false;
36
37 QThreadPool *get(const QMutexLocker<QMutex> &)
38 {
39 if (m_instance)
40 return m_instance;
41 if (m_appUnderDestruction || !qApp)
42 return nullptr;
43
44 using namespace std::chrono;
45
46 m_instance = new QThreadPool;
47 m_instance->setMaxThreadCount(1); // 1 thread;
48 static constexpr auto expiryTimeout = minutes(5);
49 m_instance->setExpiryTimeout(round<milliseconds>(d: expiryTimeout).count());
50
51 QObject::connect(qApp, signal: &QCoreApplication::aboutToQuit, context: &m_context, slot: [&] {
52 // we need to make sure that thread-local QRhi is destroyed before the application to
53 // prevent QTBUG-124189
54 QMutexLocker guard(&m_poolMutex);
55 delete m_instance;
56 m_instance = {};
57 m_appUnderDestruction = true;
58 });
59
60 QObject::connect(qApp, signal: &QCoreApplication::destroyed, context: &m_context, slot: [&] {
61 m_appUnderDestruction = false;
62 });
63 return m_instance;
64 }
65
66 template <typename Functor>
67 QFuture<void> run(Functor &&f)
68 {
69 QMutexLocker guard(&m_poolMutex);
70 QThreadPool *pool = get(guard);
71 if (!pool)
72 return QFuture<void>{};
73
74 return QtConcurrent::run(pool, std::forward<Functor>(f));
75 }
76};
77
78ThreadPoolSingleton s_threadPoolSingleton;
79
80}; // namespace
81
82template <typename Functor>
83void QGstreamerImageCapture::invokeDeferred(Functor &&fn)
84{
85 QMetaObject::invokeMethod(this, std::forward<decltype(fn)>(fn), Qt::QueuedConnection);
86}
87
88template <typename Functor>
89void QGstreamerImageCapture::runInThreadPool(Functor fn)
90{
91 int futureId = m_futureIDAllocator.fetch_add(i: 1, m: std::memory_order_relaxed);
92
93 QFuture<void> future = QtConcurrent::run([this, futureId, fn = std::move(fn)]() mutable {
94 auto cleanup = qScopeGuard([&] {
95 QMutexLocker guard(&m_pendingFuturesMutex);
96 m_pendingFutures.erase(x: futureId);
97 });
98 fn();
99 });
100
101 if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable
102 return;
103
104 QMutexLocker guard(&m_pendingFuturesMutex);
105 m_pendingFutures.emplace(args&: futureId, args: std::move(future));
106}
107
108q23::expected<QPlatformImageCapture *, QString> QGstreamerImageCapture::create(QImageCapture *parent)
109{
110 static const auto error = qGstErrorMessageIfElementsNotAvailable(
111 arg: "queue", args: "capsfilter", args: "videoconvert", args: "jpegenc", args: "jifmux", args: "fakesink");
112 if (error)
113 return q23::unexpected{ *error };
114
115 return new QGstreamerImageCapture(parent);
116}
117
118QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent)
119 : QPlatformImageCapture(parent),
120 QGstreamerBufferProbe(ProbeBuffers),
121 bin{
122 QGstBin::create(name: "imageCaptureBin"),
123 },
124 queue{
125 QGstElement::createFromFactory(factory: "queue", name: "imageCaptureQueue"),
126 },
127 filter{
128 QGstElement::createFromFactory(factory: "capsfilter", name: "filter"),
129 },
130 videoConvert{
131 QGstElement::createFromFactory(factory: "videoconvert", name: "imageCaptureConvert"),
132 },
133 encoder{
134 QGstElement::createFromFactory(factory: "jpegenc", name: "jpegEncoder"),
135 },
136 muxer{
137 QGstElement::createFromFactory(factory: "jifmux", name: "jpegMuxer"),
138 },
139 sink{
140 QGstElement::createFromFactory(factory: "fakesink", name: "imageCaptureSink"),
141 }
142{
143 // configures the queue to be fast, lightweight and non blocking
144 queue.set(property: "leaky", i: 2 /*downstream*/);
145 queue.set(property: "silent", b: true);
146 queue.set(property: "max-size-buffers", i: int(1));
147 queue.set(property: "max-size-bytes", i: int(0));
148 queue.set(property: "max-size-time", i: uint64_t(0));
149
150 bin.add(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
151 qLinkGstElements(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
152 bin.addGhostPad(child: queue, name: "sink");
153
154 addProbeToPad(pad: queue.staticPad(name: "src").pad(), downstream: false);
155
156 sink.set(property: "async", b: false);
157}
158
159QGstreamerImageCapture::~QGstreamerImageCapture()
160{
161 bin.setStateSync(state: GST_STATE_NULL);
162
163 // wait for pending futures
164 auto pendingFutures = [&] {
165 QMutexLocker guard(&m_pendingFuturesMutex);
166 return std::move(m_pendingFutures);
167 }();
168
169 for (auto &element : pendingFutures)
170 element.second.waitForFinished();
171}
172
173bool QGstreamerImageCapture::isReadyForCapture() const
174{
175 QMutexLocker guard(&m_mutex);
176 return m_session && !m_captureNextBuffer && cameraActive;
177}
178
179int QGstreamerImageCapture::capture(const QString &fileName)
180{
181 using namespace Qt::Literals;
182 QString path = QMediaStorageLocation::generateFileName(
183 requestedName: fileName, type: QStandardPaths::PicturesLocation, extension: u"jpg"_s);
184 return doCapture(fileName: std::move(path));
185}
186
187int QGstreamerImageCapture::captureToBuffer()
188{
189 return doCapture(fileName: QString());
190}
191
192int QGstreamerImageCapture::doCapture(QString fileName)
193{
194 qCDebug(qLcImageCaptureGst) << "do capture";
195
196 {
197 QMutexLocker guard(&m_mutex);
198 if (!m_session) {
199 invokeDeferred(fn: [this] {
200 emit error(id: -1, error: QImageCapture::ResourceError,
201 errorString: QPlatformImageCapture::msgImageCaptureNotSet());
202 });
203
204 qCDebug(qLcImageCaptureGst) << "error 1";
205 return -1;
206 }
207 if (!m_session->camera()) {
208 invokeDeferred(fn: [this] {
209 emit error(id: -1, error: QImageCapture::ResourceError, errorString: tr(s: "No camera available."));
210 });
211
212 qCDebug(qLcImageCaptureGst) << "error 2";
213 return -1;
214 }
215 if (m_captureNextBuffer) {
216 invokeDeferred(fn: [this] {
217 emit error(id: -1, error: QImageCapture::NotReadyError,
218 errorString: QPlatformImageCapture::msgCameraNotReady());
219 });
220
221 qCDebug(qLcImageCaptureGst) << "error 3";
222 return -1;
223 }
224 m_lastId++;
225
226 pendingImages.enqueue(t: { .id: m_lastId, .filename: std::move(fileName) });
227 // let one image pass the pipeline
228 m_captureNextBuffer = true;
229 }
230
231 emit readyForCaptureChanged(ready: false);
232 return m_lastId;
233}
234
235void QGstreamerImageCapture::saveBufferToFile(QGstBufferHandle buffer, QString filename, int taskId)
236{
237 Q_ASSERT(!filename.isEmpty());
238
239 runInThreadPool(
240 fn: [this, taskId, filename = std::move(filename), buffer = std::move(buffer)]() mutable {
241 QMutexLocker guard(&m_mutex);
242 qCDebug(qLcImageCaptureGst) << "saving image as" << filename;
243
244 QFile f(filename);
245 if (!f.open(flags: QFile::WriteOnly)) {
246 qCDebug(qLcImageCaptureGst) << " could not open image file for writing";
247 return;
248 }
249
250 GstMapInfo info;
251 if (gst_buffer_map(buffer: buffer.get(), info: &info, flags: GST_MAP_READ)) {
252 f.write(data: reinterpret_cast<const char *>(info.data), len: info.size);
253 gst_buffer_unmap(buffer: buffer.get(), info: &info);
254 }
255 f.close();
256
257 QMetaObject::invokeMethod(object: this, function: [this, taskId, filename = std::move(filename)]() mutable {
258 emit imageSaved(requestId: taskId, fileName: filename);
259 });
260 });
261}
262void QGstreamerImageCapture::convertBufferToImage(const QMutexLocker<QRecursiveMutex> &locker,
263 QGstBufferHandle buffer, QGstCaps caps,
264 QMediaMetaData metadata, int taskId)
265{
266 using namespace Qt::Literals;
267 Q_ASSERT(locker.mutex() == &m_mutex);
268 Q_ASSERT(locker.isLocked());
269
270 // QTBUG-131107: QVideoFrame::toImage() can only be called from the application thread
271 constexpr bool isOpenGLPlatform = QT_CONFIG(opengl);
272
273 // QTBUG-130970: QVideoFrame::toImage() on worker thread causes wayland to crash on the
274 // application thread
275 static const bool isWaylandQPA = QGuiApplication::platformName() == u"wayland"_s;
276
277 if (isOpenGLPlatform || isWaylandQPA) {
278 if (!m_session) {
279 qDebug() << "QGstreamerImageCapture::convertBufferToImage: no session";
280 return;
281 }
282 auto memoryFormat = caps.memoryFormat();
283
284 GstVideoInfo previewInfo;
285 QVideoFrameFormat fmt;
286 auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo();
287 if (optionalFormatAndVideoInfo)
288 std::tie(args&: fmt, args&: previewInfo) = std::move(*optionalFormatAndVideoInfo);
289
290 auto *sink = m_session->gstreamerVideoSink();
291 auto gstBuffer = std::make_unique<QGstVideoBuffer>(args: std::move(buffer), args&: previewInfo, args&: sink,
292 args&: fmt, args&: memoryFormat);
293 QVideoFrame frame = QVideoFramePrivate::createFrame(buffer: std::move(gstBuffer), format: fmt);
294
295 metadata.insert(k: QMediaMetaData::Resolution, value: frame.size());
296
297 invokeDeferred(
298 fn: [this, frame = std::move(frame), taskId, metadata = std::move(metadata)]() mutable {
299 QImage img = frame.toImage();
300 if (img.isNull()) {
301 qDebug() << "received a null image";
302 return;
303 }
304
305 emit imageExposed(requestId: taskId);
306 qCDebug(qLcImageCaptureGst) << "Image available!";
307 emit imageAvailable(requestId: taskId, buffer: frame);
308 emit imageCaptured(requestId: taskId, preview: img);
309 emit imageMetadataAvailable(id: taskId, metadata);
310 });
311 } else {
312 runInThreadPool(fn: [this, taskId, buffer = std::move(buffer), caps = std::move(caps),
313 metadata = std::move(metadata)]() mutable {
314 QMutexLocker guard(&m_mutex);
315 if (!m_session) {
316 qDebug() << "QGstreamerImageCapture::probeBuffer: no session";
317 return;
318 }
319
320 auto memoryFormat = caps.memoryFormat();
321
322 GstVideoInfo previewInfo;
323 QVideoFrameFormat fmt;
324 auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo();
325 if (optionalFormatAndVideoInfo)
326 std::tie(args&: fmt, args&: previewInfo) = std::move(*optionalFormatAndVideoInfo);
327
328 auto *sink = m_session->gstreamerVideoSink();
329 auto gstBuffer = std::make_unique<QGstVideoBuffer>(args: std::move(buffer), args&: previewInfo, args&: sink,
330 args&: fmt, args&: memoryFormat);
331
332 QVideoFrame frame = QVideoFramePrivate::createFrame(buffer: std::move(gstBuffer), format: fmt);
333 QImage img = frame.toImage();
334 if (img.isNull()) {
335 qDebug() << "received a null image";
336 return;
337 }
338
339 QMediaMetaData imageMetaData = metaData();
340 imageMetaData.insert(k: QMediaMetaData::Resolution, value: frame.size());
341
342 invokeDeferred(fn: [this, taskId, metadata = std::move(metadata), frame = std::move(frame),
343 img = std::move(img)]() mutable {
344 emit imageExposed(requestId: taskId);
345 qCDebug(qLcImageCaptureGst) << "Image available!";
346 emit imageAvailable(requestId: taskId, buffer: frame);
347 emit imageCaptured(requestId: taskId, preview: img);
348 emit imageMetadataAvailable(id: taskId, metadata);
349 });
350 });
351 }
352}
353
354void QGstreamerImageCapture::setResolution(const QSize &resolution)
355{
356 QGstCaps padCaps = bin.staticPad(name: "sink").currentCaps();
357 if (!padCaps) {
358 qDebug() << "Camera not ready";
359 return;
360 }
361 QGstCaps caps = padCaps.copy();
362 if (!caps)
363 return;
364
365 gst_caps_set_simple(caps: caps.caps(), field: "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT,
366 resolution.height(), nullptr);
367 filter.set(property: "caps", c: caps);
368}
369
370bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer)
371{
372 if (!m_captureNextBuffer.load())
373 return false;
374
375 QMutexLocker guard(&m_mutex);
376 qCDebug(qLcImageCaptureGst) << "probe buffer";
377
378 QGstBufferHandle bufferHandle{
379 buffer,
380 QGstBufferHandle::NeedsRef,
381 };
382
383 m_captureNextBuffer = false;
384
385 bool ready = isReadyForCapture();
386 invokeDeferred(fn: [this, ready] {
387 emit readyForCaptureChanged(ready);
388 });
389
390 // save file
391 PendingImage imageData = pendingImages.dequeue();
392 QString saveFileName = imageData.filename;
393 if (!saveFileName.isEmpty())
394 saveBufferToFile(buffer: bufferHandle, filename: std::move(saveFileName), taskId: imageData.id);
395
396 // convert to image and emit
397 QGstCaps caps = bin.staticPad(name: "sink").currentCaps();
398 QMediaMetaData imageMetaData = metaData();
399 convertBufferToImage(locker: guard, buffer: bufferHandle, caps: std::move(caps), metadata: std::move(imageMetaData),
400 taskId: imageData.id);
401
402 return true;
403}
404
405void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session)
406{
407 QMutexLocker guard(&m_mutex);
408 QGstreamerMediaCaptureSession *captureSession = static_cast<QGstreamerMediaCaptureSession *>(session);
409 if (m_session == captureSession)
410 return;
411
412 bool readyForCapture = isReadyForCapture();
413 if (m_session) {
414 disconnect(sender: m_session, signal: nullptr, receiver: this, member: nullptr);
415 m_lastId = 0;
416 pendingImages.clear();
417 m_captureNextBuffer = false;
418 cameraActive = false;
419 }
420
421 m_session = captureSession;
422 if (!m_session) {
423 if (readyForCapture)
424 emit readyForCaptureChanged(ready: false);
425 return;
426 }
427
428 connect(sender: m_session, signal: &QPlatformMediaCaptureSession::cameraChanged, context: this,
429 slot: &QGstreamerImageCapture::onCameraChanged);
430 onCameraChanged();
431}
432
433void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m)
434{
435 {
436 QMutexLocker guard(&m_mutex);
437 QPlatformImageCapture::setMetaData(m);
438 }
439
440 // ensure taginject injects this metaData
441 applyMetaDataToTagSetter(metadata: m, muxer);
442}
443
444void QGstreamerImageCapture::cameraActiveChanged(bool active)
445{
446 qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active;
447 if (cameraActive == active)
448 return;
449 cameraActive = active;
450 qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture();
451 emit readyForCaptureChanged(ready: isReadyForCapture());
452}
453
454void QGstreamerImageCapture::onCameraChanged()
455{
456 QMutexLocker guard(&m_mutex);
457 if (m_session->camera()) {
458 cameraActiveChanged(active: m_session->camera()->isActive());
459 connect(sender: m_session->camera(), signal: &QPlatformCamera::activeChanged, context: this,
460 slot: &QGstreamerImageCapture::cameraActiveChanged);
461 } else {
462 cameraActiveChanged(active: false);
463 }
464}
465
466QImageEncoderSettings QGstreamerImageCapture::imageSettings() const
467{
468 return m_settings;
469}
470
471void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings)
472{
473 if (m_settings != settings) {
474 QSize resolution = settings.resolution();
475 if (m_settings.resolution() != resolution && !resolution.isEmpty())
476 setResolution(resolution);
477
478 m_settings = settings;
479 }
480}
481
482QT_END_NAMESPACE
483
484#include "moc_qgstreamerimagecapture_p.cpp"
485

source code of qtmultimedia/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp