1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include "qabstractvideofilter.h"
41
42QT_BEGIN_NAMESPACE
43
44/*!
45 \class QAbstractVideoFilter
46 \since 5.5
47 \brief The QAbstractVideoFilter class represents a filter that is applied to the video frames
48 received by a VideoOutput type.
49 \inmodule QtMultimedia
50
51 \ingroup multimedia
52 \ingroup multimedia_video
53
54 QAbstractVideoFilter provides a convenient way for applications to run image
55 processing, computer vision algorithms or any generic transformation or
56 calculation on the output of a VideoOutput type, regardless of the source
57 (video or camera). By providing a simple interface it allows applications and
58 third parties to easily develop QML types that provide image processing
59 algorithms using popular frameworks like \l{http://opencv.org}{OpenCV}. Due to
60 the close integration with the final stages of the Qt Multimedia video
61 pipeline, accelerated and possibly zero-copy solutions are feasible too: for
62 instance, a plugin providing OpenCL-based algorithms can use OpenCL's OpenGL
63 interop to use the OpenGL textures created by a hardware accelerated video
64 decoder, without additional readbacks and copies.
65
66 \note QAbstractVideoFilter is not always the best choice. To apply effects or
67 transformations using OpenGL shaders to the image shown on screen, the
68 standard Qt Quick approach of using ShaderEffect items in combination with
69 VideoOutput should be used. VideoFilter is not a replacement for this. It is
70 rather targeted for performing computations (that do not necessarily change
71 the image shown on screen) and computer vision algorithms provided by
72 external frameworks.
73
74 QAbstractVideoFilter is meant to be subclassed. The subclasses are then registered to
75 the QML engine, so they can be used as a QML type. The list of filters are
76 assigned to a VideoOutput type via its \l{QtMultimedia::VideoOutput::filters}{filters}
77 property.
78
79 A single filter represents one transformation or processing step on
80 a video frame. The output is a modified video frame, some arbitrary data or
81 both. For example, image transformations will result in a different image,
82 whereas an algorithm for detecting objects on an image will likely provide
83 a list of rectangles.
84
85 Arbitrary data can be represented as properties on the QAbstractVideoFilter subclass
86 and on the QObject or QJSValue instances passed to its signals. What exactly
87 these properties and signals are, is up to the individual video
88 filters. Completion of the operations can be indicated by
89 signals. Computations that do not result in a modified image will pass the
90 input image through so that subsequent filters can be placed after them.
91
92 Properties set on QAbstractVideoFilter serve as input to the computation, similarly
93 to how uniform values are specified in ShaderEffect types. The changed
94 property values are taken into use when the next video frame is processed.
95
96 The typical usage is to subclass QAbstractVideoFilter and QVideoFilterRunnable:
97
98 \badcode
99 class MyFilterRunnable : public QVideoFilterRunnable {
100 public:
101 QVideoFrame run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags) { ... }
102 };
103
104 class MyFilter : public QAbstractVideoFilter {
105 public:
106 QVideoFilterRunnable *createFilterRunnable() { return new MyFilterRunnable; }
107 signals:
108 void finished(QObject *result);
109 };
110
111 int main(int argc, char **argv) {
112 ...
113 qmlRegisterType<MyFilter>("my.uri", 1, 0, "MyFilter");
114 ...
115 }
116 \endcode
117
118 MyFilter is thus accessible from QML:
119
120 \badcode
121 import my.uri 1.0
122
123 Camera {
124 id: camera
125 }
126 MyFilter {
127 id: filter
128 // set properties, they can also be animated
129 onFinished: console.log("results of the computation: " + result)
130 }
131 VideoOutput {
132 source: camera
133 filters: [ filter ]
134 anchors.fill: parent
135 }
136 \endcode
137
138 This also allows providing filters in QML plugins, separately from the application.
139
140 \sa VideoOutput, Camera, MediaPlayer, QVideoFilterRunnable
141*/
142
143/*!
144 \class QVideoFilterRunnable
145 \since 5.5
146 \brief The QVideoFilterRunnable class represents the implementation of a filter
147 that owns all graphics and computational resources, and performs the actual filtering
148 or calculations.
149 \inmodule QtMultimedia
150
151 \ingroup multimedia
152 \ingroup multimedia_video
153
154 Video filters are split into QAbstractVideoFilter and corresponding QVideoFilterRunnable
155 instances, similar to QQuickItem and QSGNode. This is necessary to support
156 threaded rendering scenarios. When using the threaded render loop of the Qt
157 Quick scene graph, all rendering happens on a dedicated thread.
158 QVideoFilterRunnable instances always live on this thread and all its functions,
159 run(), the constructor, and the destructor, are guaranteed to be invoked on
160 that thread with the OpenGL context bound. QAbstractVideoFilter instances live on
161 the main (GUI) thread, like any other QObject and QQuickItem instances
162 created from QML.
163
164 Once created, QVideoFilterRunnable instances are managed by Qt Multimedia and
165 will be automatically destroyed and recreated when necessary, for example
166 when the scene graph is invalidated or the QQuickWindow changes or is closed.
167 Creation happens via the QAbstractVideoFilter::createFilterRunnable() factory function.
168
169 \sa QAbstractVideoFilter
170 */
171
172/*!
173 \fn QVideoFrame QVideoFilterRunnable::run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags)
174
175 Reimplement this function to perform filtering or computation on the \a
176 input video frame. Like the constructor and destructor, this function is
177 always called on the render thread with the OpenGL context bound.
178
179 Implementations that do not modify the video frame can simply return \a input.
180
181 It is safe to access properties of the associated QAbstractVideoFilter instance from
182 this function.
183
184 \a input will not be mapped, it is up to this function to call QVideoFrame::map()
185 and QVideoFrame::unmap() as necessary.
186
187 \a surfaceFormat provides additional information, for example it can be used
188 to determine which way is up in the input image as that is important for
189 filters to operate on multiple platforms with multiple cameras.
190
191 \a flags contains additional information about the filter's invocation. For
192 example the LastInChain flag indicates that the filter is the last in a
193 VideoOutput's associated filter list. This can be very useful in cases where
194 multiple filters are chained together and the work is performed on image data
195 in some custom format (for example a format specific to some computer vision
196 framework). To avoid conversion on every filter in the chain, all
197 intermediate filters can return a QVideoFrame hosting data in the custom
198 format. Only the last, where the flag is set, returns a QVideoFrame in a
199 format compatible with Qt.
200
201 Filters that want to expose the results of their computation to Javascript
202 code in QML can declare their own custom signals in the QAbstractVideoFilter
203 subclass to indicate the completion of the operation. For filters that only
204 calculate some results and do not modify the video frame, it is also possible
205 to operate asynchronously. They can queue the necessary operations using the
206 compute API and return from this function without emitting any signals. The
207 signal indicating the completion is then emitted only when the compute API
208 indicates that the operations were done and the results are available. Note
209 that it is strongly recommended to represent the filter's output data as a
210 separate instance of QJSValue or a QObject-derived class which is passed as a
211 parameter to the signal and becomes exposed to the Javascript engine. In case
212 of QObject the ownership of this object is controlled by the standard QML
213 rules: if it has no parent, ownership is transferred to the Javascript engine,
214 otherwise it stays with the emitter. Note that the signal connection may be
215 queued,for example when using the threaded render loop of Qt Quick, and so the
216 object must stay valid for a longer time, destroying it right after calling
217 this function is not safe. Using a dedicated results object is guaranteed to
218 be safe even when using threaded rendering. The same is not necessarily true
219 for properties on the QAbstractVideoFilter instance itself: properties can
220 safely be read in run() since the gui thread is blocked during that time but
221 writing may become problematic.
222
223 \note Avoid time consuming operations in this function as they block the
224 entire rendering of the application.
225
226 \note The handleType() and pixelFormat() of \a input is completely up to the
227 video decoding backend on the platform in use. On some platforms different
228 forms of input are used depending on the graphics stack. For example, when
229 playing back videos on Windows with the WMF backend, QVideoFrame contains
230 OpenGL-wrapped Direct3D textures in case of using ANGLE, but regular pixel
231 data when using desktop OpenGL (opengl32.dll). Similarly, the video file
232 format will often decide if the data is RGB or YUV, but this may also depend
233 on the decoder and the configuration in use. The returned video frame does
234 not have to be in the same format as the input, for example a filter with an
235 input of a QVideoFrame backed by system memory can output a QVideoFrame with
236 an OpenGL texture handle.
237
238 \sa QVideoFrame, QVideoSurfaceFormat
239 */
240
241/*!
242 \enum QVideoFilterRunnable::RunFlag
243
244 \value LastInChain Indicates that the filter runnable's associated QAbstractVideoFilter
245 is the last in the corresponding VideoOutput type's filters list, meaning
246 that the returned frame is the one that is going to be presented to the scene
247 graph without invoking any further filters.
248 */
249
250class QAbstractVideoFilterPrivate
251{
252public:
253 QAbstractVideoFilterPrivate() :
254 active(true)
255 { }
256
257 bool active;
258};
259
260/*!
261 \internal
262 */
263QVideoFilterRunnable::~QVideoFilterRunnable()
264{
265}
266
267/*!
268 Constructs a new QAbstractVideoFilter instance with parent object \a parent.
269 */
270QAbstractVideoFilter::QAbstractVideoFilter(QObject *parent) :
271 QObject(parent),
272 d_ptr(new QAbstractVideoFilterPrivate)
273{
274}
275
276/*!
277 \internal
278 */
279QAbstractVideoFilter::~QAbstractVideoFilter()
280{
281 delete d_ptr;
282}
283
284/*!
285 \property QAbstractVideoFilter::active
286 \brief the active status of the filter.
287
288 This is true if the filter is active, false otherwise.
289
290 By default filters are active. When set to \c false, the filter will be
291 ignored by the VideoOutput type.
292 */
293bool QAbstractVideoFilter::isActive() const
294{
295 Q_D(const QAbstractVideoFilter);
296 return d->active;
297}
298
299void QAbstractVideoFilter::setActive(bool v)
300{
301 Q_D(QAbstractVideoFilter);
302 if (d->active != v) {
303 d->active = v;
304 emit activeChanged();
305 }
306}
307
308/*!
309 \fn QVideoFilterRunnable *QAbstractVideoFilter::createFilterRunnable()
310
311 Factory function to create a new instance of a QVideoFilterRunnable subclass
312 corresponding to this filter.
313
314 This function is called on the thread on which the Qt Quick scene graph
315 performs rendering, with the OpenGL context bound. Ownership of the returned
316 instance is transferred: the returned instance will live on the render thread
317 and will be destroyed automatically when necessary.
318
319 Typically, implementations of the function will simply construct a new
320 QVideoFilterRunnable instance, passing \c this to the constructor as the
321 filter runnables must know their associated QAbstractVideoFilter instance to
322 access dynamic properties and optionally emit signals.
323 */
324
325QT_END_NAMESPACE
326

source code of qtmultimedia/src/multimedia/video/qabstractvideofilter.cpp