1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstreamerimagecapture_p.h"
5
6#include <QtMultimedia/qvideoframeformat.h>
7#include <QtMultimedia/private/qmediastoragelocation_p.h>
8#include <QtMultimedia/private/qplatformcamera_p.h>
9#include <QtMultimedia/private/qplatformimagecapture_p.h>
10#include <QtMultimedia/private/qvideoframe_p.h>
11#include <QtGui/qguiapplication.h>
12#include <QtCore/qdebug.h>
13#include <QtCore/qdir.h>
14#include <QtCore/qstandardpaths.h>
15#include <QtCore/qcoreapplication.h>
16#include <QtCore/qloggingcategory.h>
17
18#include <common/qgstreamermetadata_p.h>
19#include <common/qgstvideobuffer_p.h>
20#include <common/qgstutils_p.h>
21
22#include <utility>
23
24QT_BEGIN_NAMESPACE
25
26namespace {
27Q_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture")
28
29struct ThreadPoolSingleton
30{
31 QObject m_context;
32 QMutex m_poolMutex;
33 QThreadPool *m_instance{};
34 bool m_appUnderDestruction = false;
35
36 QThreadPool *get(const QMutexLocker<QMutex> &)
37 {
38 if (m_instance)
39 return m_instance;
40 if (m_appUnderDestruction || !qApp)
41 return nullptr;
42
43 using namespace std::chrono;
44
45 m_instance = new QThreadPool;
46 m_instance->setMaxThreadCount(1); // 1 thread;
47 static constexpr auto expiryTimeout = minutes(5);
48 m_instance->setExpiryTimeout(round<milliseconds>(d: expiryTimeout).count());
49
50 QObject::connect(qApp, signal: &QCoreApplication::aboutToQuit, context: &m_context, slot: [&] {
51 // we need to make sure that thread-local QRhi is destroyed before the application to
52 // prevent QTBUG-124189
53 QMutexLocker guard(&m_poolMutex);
54 delete m_instance;
55 m_instance = {};
56 m_appUnderDestruction = true;
57 });
58
59 QObject::connect(qApp, signal: &QCoreApplication::destroyed, context: &m_context, slot: [&] {
60 m_appUnderDestruction = false;
61 });
62 return m_instance;
63 }
64
65 template <typename Functor>
66 QFuture<void> run(Functor &&f)
67 {
68 QMutexLocker guard(&m_poolMutex);
69 QThreadPool *pool = get(guard);
70 if (!pool)
71 return QFuture<void>{};
72
73 return QtConcurrent::run(pool, std::forward<Functor>(f));
74 }
75};
76
77ThreadPoolSingleton s_threadPoolSingleton;
78
79}; // namespace
80
81template <typename Functor>
82void QGstreamerImageCapture::invokeDeferred(Functor &&fn)
83{
84 QMetaObject::invokeMethod(this, std::forward<decltype(fn)>(fn), Qt::QueuedConnection);
85}
86
87template <typename Functor>
88void QGstreamerImageCapture::runInThreadPool(Functor fn)
89{
90 int futureId = m_futureIDAllocator.fetch_add(i: 1, m: std::memory_order_relaxed);
91
92 QFuture<void> future = QtConcurrent::run([this, futureId, fn = std::move(fn)]() mutable {
93 auto cleanup = qScopeGuard([&] {
94 QMutexLocker guard(&m_pendingFuturesMutex);
95 m_pendingFutures.erase(x: futureId);
96 });
97 fn();
98 });
99
100 if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable
101 return;
102
103 QMutexLocker guard(&m_pendingFuturesMutex);
104 m_pendingFutures.emplace(args&: futureId, args: std::move(future));
105}
106
107QMaybe<QPlatformImageCapture *> QGstreamerImageCapture::create(QImageCapture *parent)
108{
109 static const auto error = qGstErrorMessageIfElementsNotAvailable(
110 arg: "queue", args: "capsfilter", args: "videoconvert", args: "jpegenc", args: "jifmux", args: "fakesink");
111 if (error)
112 return *error;
113
114 return new QGstreamerImageCapture(parent);
115}
116
117QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent)
118 : QPlatformImageCapture(parent),
119 QGstreamerBufferProbe(ProbeBuffers),
120 bin{
121 QGstBin::create(name: "imageCaptureBin"),
122 },
123 queue{
124 QGstElement::createFromFactory(factory: "queue", name: "imageCaptureQueue"),
125 },
126 filter{
127 QGstElement::createFromFactory(factory: "capsfilter", name: "filter"),
128 },
129 videoConvert{
130 QGstElement::createFromFactory(factory: "videoconvert", name: "imageCaptureConvert"),
131 },
132 encoder{
133 QGstElement::createFromFactory(factory: "jpegenc", name: "jpegEncoder"),
134 },
135 muxer{
136 QGstElement::createFromFactory(factory: "jifmux", name: "jpegMuxer"),
137 },
138 sink{
139 QGstElement::createFromFactory(factory: "fakesink", name: "imageCaptureSink"),
140 }
141{
142 // configures the queue to be fast, lightweight and non blocking
143 queue.set(property: "leaky", i: 2 /*downstream*/);
144 queue.set(property: "silent", b: true);
145 queue.set(property: "max-size-buffers", i: int(1));
146 queue.set(property: "max-size-bytes", i: int(0));
147 queue.set(property: "max-size-time", i: uint64_t(0));
148
149 bin.add(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
150 qLinkGstElements(ts: queue, ts: filter, ts: videoConvert, ts: encoder, ts: muxer, ts: sink);
151 bin.addGhostPad(child: queue, name: "sink");
152
153 addProbeToPad(pad: queue.staticPad(name: "src").pad(), downstream: false);
154
155 sink.set(property: "async", b: false);
156}
157
158QGstreamerImageCapture::~QGstreamerImageCapture()
159{
160 bin.setStateSync(state: GST_STATE_NULL);
161
162 // wait for pending futures
163 auto pendingFutures = [&] {
164 QMutexLocker guard(&m_pendingFuturesMutex);
165 return std::move(m_pendingFutures);
166 }();
167
168 for (auto &element : pendingFutures)
169 element.second.waitForFinished();
170}
171
172bool QGstreamerImageCapture::isReadyForCapture() const
173{
174 QMutexLocker guard(&m_mutex);
175 return m_session && !m_captureNextBuffer && cameraActive;
176}
177
178int QGstreamerImageCapture::capture(const QString &fileName)
179{
180 using namespace Qt::Literals;
181 QString path = QMediaStorageLocation::generateFileName(
182 requestedName: fileName, type: QStandardPaths::PicturesLocation, extension: u"jpg"_s);
183 return doCapture(fileName: std::move(path));
184}
185
186int QGstreamerImageCapture::captureToBuffer()
187{
188 return doCapture(fileName: QString());
189}
190
191int QGstreamerImageCapture::doCapture(QString fileName)
192{
193 qCDebug(qLcImageCaptureGst) << "do capture";
194
195 {
196 QMutexLocker guard(&m_mutex);
197 if (!m_session) {
198 invokeDeferred(fn: [this] {
199 emit error(id: -1, error: QImageCapture::ResourceError,
200 errorString: QPlatformImageCapture::msgImageCaptureNotSet());
201 });
202
203 qCDebug(qLcImageCaptureGst) << "error 1";
204 return -1;
205 }
206 if (!m_session->camera()) {
207 invokeDeferred(fn: [this] {
208 emit error(id: -1, error: QImageCapture::ResourceError, errorString: tr(s: "No camera available."));
209 });
210
211 qCDebug(qLcImageCaptureGst) << "error 2";
212 return -1;
213 }
214 if (m_captureNextBuffer) {
215 invokeDeferred(fn: [this] {
216 emit error(id: -1, error: QImageCapture::NotReadyError,
217 errorString: QPlatformImageCapture::msgCameraNotReady());
218 });
219
220 qCDebug(qLcImageCaptureGst) << "error 3";
221 return -1;
222 }
223 m_lastId++;
224
225 pendingImages.enqueue(t: { .id: m_lastId, .filename: std::move(fileName) });
226 // let one image pass the pipeline
227 m_captureNextBuffer = true;
228 }
229
230 emit readyForCaptureChanged(ready: false);
231 return m_lastId;
232}
233
234void QGstreamerImageCapture::saveBufferToFile(QGstBufferHandle buffer, QString filename, int taskId)
235{
236 Q_ASSERT(!filename.isEmpty());
237
238 runInThreadPool(
239 fn: [this, taskId, filename = std::move(filename), buffer = std::move(buffer)]() mutable {
240 QMutexLocker guard(&m_mutex);
241 qCDebug(qLcImageCaptureGst) << "saving image as" << filename;
242
243 QFile f(filename);
244 if (!f.open(flags: QFile::WriteOnly)) {
245 qCDebug(qLcImageCaptureGst) << " could not open image file for writing";
246 return;
247 }
248
249 GstMapInfo info;
250 if (gst_buffer_map(buffer: buffer.get(), info: &info, flags: GST_MAP_READ)) {
251 f.write(data: reinterpret_cast<const char *>(info.data), len: info.size);
252 gst_buffer_unmap(buffer: buffer.get(), info: &info);
253 }
254 f.close();
255
256 QMetaObject::invokeMethod(object: this, function: [this, taskId, filename = std::move(filename)]() mutable {
257 emit imageSaved(requestId: taskId, fileName: filename);
258 });
259 });
260}
261void QGstreamerImageCapture::convertBufferToImage(const QMutexLocker<QRecursiveMutex> &locker,
262 QGstBufferHandle buffer, QGstCaps caps,
263 QMediaMetaData metadata, int taskId)
264{
265 using namespace Qt::Literals;
266 Q_ASSERT(locker.mutex() == &m_mutex);
267 Q_ASSERT(locker.isLocked());
268
269 // QTBUG-131107: QVideoFrame::toImage() can only be called from the application thread
270 constexpr bool isOpenGLPlatform = QT_CONFIG(opengl);
271
272 // QTBUG-130970: QVideoFrame::toImage() on worker thread causes wayland to crash on the
273 // application thread
274 static const bool isWaylandQPA = QGuiApplication::platformName() == u"wayland"_s;
275
276 if (isOpenGLPlatform || isWaylandQPA) {
277 if (!m_session) {
278 qDebug() << "QGstreamerImageCapture::convertBufferToImage: no session";
279 return;
280 }
281 auto memoryFormat = caps.memoryFormat();
282
283 GstVideoInfo previewInfo;
284 QVideoFrameFormat fmt;
285 auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo();
286 if (optionalFormatAndVideoInfo)
287 std::tie(args&: fmt, args&: previewInfo) = std::move(*optionalFormatAndVideoInfo);
288
289 auto *sink = m_session->gstreamerVideoSink();
290 auto gstBuffer = std::make_unique<QGstVideoBuffer>(args: std::move(buffer), args&: previewInfo, args&: sink,
291 args&: fmt, args&: memoryFormat);
292 QVideoFrame frame = QVideoFramePrivate::createFrame(buffer: std::move(gstBuffer), format: fmt);
293
294 metadata.insert(k: QMediaMetaData::Resolution, value: frame.size());
295
296 invokeDeferred(
297 fn: [this, frame = std::move(frame), taskId, metadata = std::move(metadata)]() mutable {
298 QImage img = frame.toImage();
299 if (img.isNull()) {
300 qDebug() << "received a null image";
301 return;
302 }
303
304 emit imageExposed(requestId: taskId);
305 qCDebug(qLcImageCaptureGst) << "Image available!";
306 emit imageAvailable(requestId: taskId, buffer: frame);
307 emit imageCaptured(requestId: taskId, preview: img);
308 emit imageMetadataAvailable(id: taskId, metadata);
309 });
310 } else {
311 runInThreadPool(fn: [this, taskId, buffer = std::move(buffer), caps = std::move(caps),
312 metadata = std::move(metadata)]() mutable {
313 QMutexLocker guard(&m_mutex);
314 if (!m_session) {
315 qDebug() << "QGstreamerImageCapture::probeBuffer: no session";
316 return;
317 }
318
319 auto memoryFormat = caps.memoryFormat();
320
321 GstVideoInfo previewInfo;
322 QVideoFrameFormat fmt;
323 auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo();
324 if (optionalFormatAndVideoInfo)
325 std::tie(args&: fmt, args&: previewInfo) = std::move(*optionalFormatAndVideoInfo);
326
327 auto *sink = m_session->gstreamerVideoSink();
328 auto gstBuffer = std::make_unique<QGstVideoBuffer>(args: std::move(buffer), args&: previewInfo, args&: sink,
329 args&: fmt, args&: memoryFormat);
330
331 QVideoFrame frame = QVideoFramePrivate::createFrame(buffer: std::move(gstBuffer), format: fmt);
332 QImage img = frame.toImage();
333 if (img.isNull()) {
334 qDebug() << "received a null image";
335 return;
336 }
337
338 QMediaMetaData imageMetaData = metaData();
339 imageMetaData.insert(k: QMediaMetaData::Resolution, value: frame.size());
340
341 invokeDeferred(fn: [this, taskId, metadata = std::move(metadata), frame = std::move(frame),
342 img = std::move(img)]() mutable {
343 emit imageExposed(requestId: taskId);
344 qCDebug(qLcImageCaptureGst) << "Image available!";
345 emit imageAvailable(requestId: taskId, buffer: frame);
346 emit imageCaptured(requestId: taskId, preview: img);
347 emit imageMetadataAvailable(id: taskId, metadata);
348 });
349 });
350 }
351}
352
353void QGstreamerImageCapture::setResolution(const QSize &resolution)
354{
355 QGstCaps padCaps = bin.staticPad(name: "sink").currentCaps();
356 if (!padCaps) {
357 qDebug() << "Camera not ready";
358 return;
359 }
360 QGstCaps caps = padCaps.copy();
361 if (!caps)
362 return;
363
364 gst_caps_set_simple(caps: caps.caps(), field: "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT,
365 resolution.height(), nullptr);
366 filter.set(property: "caps", c: caps);
367}
368
369bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer)
370{
371 if (!m_captureNextBuffer.load())
372 return false;
373
374 QMutexLocker guard(&m_mutex);
375 qCDebug(qLcImageCaptureGst) << "probe buffer";
376
377 QGstBufferHandle bufferHandle{
378 buffer,
379 QGstBufferHandle::NeedsRef,
380 };
381
382 m_captureNextBuffer = false;
383
384 bool ready = isReadyForCapture();
385 invokeDeferred(fn: [this, ready] {
386 emit readyForCaptureChanged(ready);
387 });
388
389 // save file
390 PendingImage imageData = pendingImages.dequeue();
391 QString saveFileName = imageData.filename;
392 if (!saveFileName.isEmpty())
393 saveBufferToFile(buffer: bufferHandle, filename: std::move(saveFileName), taskId: imageData.id);
394
395 // convert to image and emit
396 QGstCaps caps = bin.staticPad(name: "sink").currentCaps();
397 QMediaMetaData imageMetaData = metaData();
398 convertBufferToImage(locker: guard, buffer: bufferHandle, caps: std::move(caps), metadata: std::move(imageMetaData),
399 taskId: imageData.id);
400
401 return true;
402}
403
404void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session)
405{
406 QMutexLocker guard(&m_mutex);
407 QGstreamerMediaCaptureSession *captureSession = static_cast<QGstreamerMediaCaptureSession *>(session);
408 if (m_session == captureSession)
409 return;
410
411 bool readyForCapture = isReadyForCapture();
412 if (m_session) {
413 disconnect(sender: m_session, signal: nullptr, receiver: this, member: nullptr);
414 m_lastId = 0;
415 pendingImages.clear();
416 m_captureNextBuffer = false;
417 cameraActive = false;
418 }
419
420 m_session = captureSession;
421 if (!m_session) {
422 if (readyForCapture)
423 emit readyForCaptureChanged(ready: false);
424 return;
425 }
426
427 connect(sender: m_session, signal: &QPlatformMediaCaptureSession::cameraChanged, context: this,
428 slot: &QGstreamerImageCapture::onCameraChanged);
429 onCameraChanged();
430}
431
432void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m)
433{
434 {
435 QMutexLocker guard(&m_mutex);
436 QPlatformImageCapture::setMetaData(m);
437 }
438
439 // ensure taginject injects this metaData
440 applyMetaDataToTagSetter(metadata: m, muxer);
441}
442
443void QGstreamerImageCapture::cameraActiveChanged(bool active)
444{
445 qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active;
446 if (cameraActive == active)
447 return;
448 cameraActive = active;
449 qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture();
450 emit readyForCaptureChanged(ready: isReadyForCapture());
451}
452
453void QGstreamerImageCapture::onCameraChanged()
454{
455 QMutexLocker guard(&m_mutex);
456 if (m_session->camera()) {
457 cameraActiveChanged(active: m_session->camera()->isActive());
458 connect(sender: m_session->camera(), signal: &QPlatformCamera::activeChanged, context: this,
459 slot: &QGstreamerImageCapture::cameraActiveChanged);
460 } else {
461 cameraActiveChanged(active: false);
462 }
463}
464
465QImageEncoderSettings QGstreamerImageCapture::imageSettings() const
466{
467 return m_settings;
468}
469
470void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings)
471{
472 if (m_settings != settings) {
473 QSize resolution = settings.resolution();
474 if (m_settings.resolution() != resolution && !resolution.isEmpty())
475 setResolution(resolution);
476
477 m_settings = settings;
478 }
479}
480
481QT_END_NAMESPACE
482
483#include "moc_qgstreamerimagecapture_p.cpp"
484

source code of qtmultimedia/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp