1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qvideoframe.h"
5
6#include "qvideoframe_p.h"
7#include "qvideotexturehelper_p.h"
8#include "qmultimediautils_p.h"
9#include "qmemoryvideobuffer_p.h"
10#include "qvideoframeconverter_p.h"
11#include "qimagevideobuffer_p.h"
12#include "qpainter.h"
13#include <qtextlayout.h>
14
15#include <qimage.h>
16#include <qpair.h>
17#include <qsize.h>
18#include <qvariant.h>
19#include <rhi/qrhi.h>
20
21#include <QDebug>
22
23QT_BEGIN_NAMESPACE
24
25QT_DEFINE_QESDP_SPECIALIZATION_DTOR(QVideoFramePrivate);
26
27/*!
28 \class QVideoFrame
29 \brief The QVideoFrame class represents a frame of video data.
30 \inmodule QtMultimedia
31
32 \ingroup multimedia
33 \ingroup multimedia_video
34
35 A QVideoFrame encapsulates the pixel data of a video frame, and information about the frame.
36
37 Video frames can come from several places - decoded \l {QMediaPlayer}{media}, a
38 \l {QCamera}{camera}, or generated programmatically. The way pixels are described in these
39 frames can vary greatly, and some pixel formats offer greater compression opportunities at
40 the expense of ease of use.
41
42 The pixel contents of a video frame can be mapped to memory using the map() function. After
43 a successful call to map(), the video data can be accessed through various functions. Some of
44 the YUV pixel formats provide the data in several planes. The planeCount() method will return
45 the amount of planes that being used.
46
47 While mapped, the video data of each plane can accessed using the bits() function, which
48 returns a pointer to a buffer. The size of this buffer is given by the mappedBytes() function,
49 and the size of each line is given by bytesPerLine(). The return value of the handle()
50 function may also be used to access frame data using the internal buffer's native APIs
51 (for example - an OpenGL texture handle).
52
53 A video frame can also have timestamp information associated with it. These timestamps can be
54 used to determine when to start and stop displaying the frame.
55
56 QVideoFrame objects can consume a significant amount of memory or system resources and
57 should not be held for longer than required by the application.
58
59 \note Since video frames can be expensive to copy, QVideoFrame is explicitly shared, so any
60 change made to a video frame will also apply to any copies.
61
62 \sa QAbstractVideoBuffer, QVideoFrameFormat, QVideoFrame::MapMode
63*/
64
65/*!
66 \enum QVideoFrame::HandleType
67
68 Identifies the type of a video buffers handle.
69
70 \value NoHandle
71 The buffer has no handle, its data can only be accessed by mapping the buffer.
72 \value RhiTextureHandle
73 The handle of the buffer is defined by The Qt Rendering Hardware Interface
74 (RHI). RHI is Qt's internal graphics abstraction for 3D APIs, such as
75 OpenGL, Vulkan, Metal, and Direct 3D.
76
77 \sa handleType()
78*/
79
80
81/*!
82 Constructs a null video frame.
83*/
84QVideoFrame::QVideoFrame()
85{
86}
87
88#if QT_DEPRECATED_SINCE(6, 8)
89
90/*!
91 \internal
92 Constructs a video frame from a \a buffer with the given pixel \a format and \a size in pixels.
93
94 \note This doesn't increment the reference count of the video buffer.
95*/
96QVideoFrame::QVideoFrame(QAbstractVideoBuffer *buffer, const QVideoFrameFormat &format)
97 : d(new QVideoFramePrivate(format, std::unique_ptr<QAbstractVideoBuffer>(buffer)))
98{
99}
100
101/*!
102 \internal
103*/
104QAbstractVideoBuffer *QVideoFrame::videoBuffer() const
105{
106 return d ? d->videoBuffer.get() : nullptr;
107}
108
109#endif
110
111/*!
112 Constructs a video frame of the given pixel \a format.
113
114*/
115QVideoFrame::QVideoFrame(const QVideoFrameFormat &format)
116 : d(new QVideoFramePrivate(format))
117{
118 auto *textureDescription = QVideoTextureHelper::textureDescription(format: format.pixelFormat());
119 qsizetype bytes = textureDescription->bytesForSize(s: format.frameSize());
120 if (bytes > 0) {
121 QByteArray data;
122 data.resize(size: bytes);
123
124 // Check the memory was successfully allocated.
125 if (!data.isEmpty())
126 d->videoBuffer = std::make_unique<QMemoryVideoBuffer>(
127 args&: data, args: textureDescription->strideForWidth(width: format.frameWidth()));
128 }
129}
130
131/*!
132 Constructs a QVideoFrame from a QImage.
133 \since 6.8
134
135 If the QImage::Format matches one of the formats in
136 QVideoFrameFormat::PixelFormat, the QVideoFrame will hold an instance of
137 the \a image and use that format without any pixel format conversion.
138 In this case, pixel data will be copied only if you call \l{QVideoFrame::map}
139 with \c WriteOnly flag while keeping the original image.
140
141 Otherwise, if the QImage::Format matches none of video formats,
142 the image is first converted to a supported (A)RGB format using
143 QImage::convertedTo() with the Qt::AutoColor flag.
144 This may incur a performance penalty.
145
146 If QImage::isNull() evaluates to true for the input QImage, the
147 QVideoFrame will be invalid and QVideoFrameFormat::isValid() will
148 return false.
149
150 \sa QVideoFrameFormat::pixelFormatFromImageFormat()
151 \sa QImage::convertedTo()
152 \sa QImage::isNull()
153*/
154QVideoFrame::QVideoFrame(const QImage &image)
155{
156 auto buffer = std::make_unique<QImageVideoBuffer>(args: image);
157
158 // If the QImage::Format is not convertible to QVideoFrameFormat,
159 // QImageVideoBuffer automatically converts image to a compatible
160 // (A)RGB format.
161 const QImage &bufferImage = buffer->underlyingImage();
162
163 if (bufferImage.isNull())
164 return;
165
166 // `bufferImage` is now supported by QVideoFrameFormat::pixelFormatFromImageFormat()
167 QVideoFrameFormat format = {
168 bufferImage.size(), QVideoFrameFormat::pixelFormatFromImageFormat(format: bufferImage.format())
169 };
170
171 Q_ASSERT(format.isValid());
172
173 d = new QVideoFramePrivate{ std::move(format), std::move(buffer) };
174}
175
176/*!
177 Constructs a QVideoFrame from a \l QAbstractVideoBuffer.
178
179 \since 6.8
180
181 The specified \a videoBuffer refers to an instance a reimplemented
182 \l QAbstractVideoBuffer. The instance is expected to contain a preallocated custom
183 video buffer and must implement \l QAbstractVideoBuffer::format,
184 \l QAbstractVideoBuffer::map, and \l QAbstractVideoBuffer::unmap for GPU content.
185
186 If \a videoBuffer is null or gets an invalid \l QVideoFrameFormat,
187 the constructors creates an invalid video frame.
188
189 The created frame will hold ownership of the specified video buffer for its lifetime.
190 Considering that QVideoFrame is implemented via a shared private object,
191 the specified video buffer will be destroyed upon destruction of the last copy
192 of the created video frame.
193
194 Note, if a video frame has been passed to \l QMediaRecorder or a rendering pipeline,
195 the lifetime of the frame is undefined, and the media recorder can destroy it
196 in a different thread.
197
198 QVideoFrame will contain own instance of QVideoFrameFormat.
199 Upon invoking \l setStreamFrameRate, \l setMirrored, or \l setRotation,
200 the inner format can be modified, and \l surfaceFormat will return
201 a detached instance.
202
203 \sa QAbstractVideoBuffer, QVideoFrameFormat
204*/
205QVideoFrame::QVideoFrame(std::unique_ptr<QAbstractVideoBuffer> videoBuffer)
206{
207 if (!videoBuffer)
208 return;
209
210 QVideoFrameFormat format = videoBuffer->format();
211 if (!format.isValid())
212 return;
213
214 d = new QVideoFramePrivate{ std::move(format), std::move(videoBuffer) };
215}
216
217/*!
218 Constructs a shallow copy of \a other. Since QVideoFrame is
219 explicitly shared, these two instances will reflect the same frame.
220
221*/
222QVideoFrame::QVideoFrame(const QVideoFrame &other) = default;
223
224/*!
225 \fn QVideoFrame::QVideoFrame(QVideoFrame &&other)
226
227 Constructs a QVideoFrame by moving from \a other.
228*/
229
230/*!
231 \fn void QVideoFrame::swap(QVideoFrame &other) noexcept
232
233 Swaps the current video frame with \a other.
234*/
235
236/*!
237 \fn QVideoFrame &QVideoFrame::operator=(QVideoFrame &&other)
238
239 Moves \a other into this QVideoFrame.
240*/
241
242/*!
243 Assigns the contents of \a other to this video frame. Since QVideoFrame is
244 explicitly shared, these two instances will reflect the same frame.
245
246*/
247QVideoFrame &QVideoFrame::operator =(const QVideoFrame &other) = default;
248
249/*!
250 \return \c true if this QVideoFrame and \a other reflect the same frame.
251 */
252bool QVideoFrame::operator==(const QVideoFrame &other) const
253{
254 // Due to explicit sharing we just compare the QSharedData which in turn compares the pointers.
255 return d == other.d;
256}
257
258/*!
259 \return \c true if this QVideoFrame and \a other do not reflect the same frame.
260 */
261bool QVideoFrame::operator!=(const QVideoFrame &other) const
262{
263 return d != other.d;
264}
265
266/*!
267 Destroys a video frame.
268*/
269QVideoFrame::~QVideoFrame() = default;
270
271/*!
272 Identifies whether a video frame is valid.
273
274 An invalid frame has no video buffer associated with it.
275
276 Returns true if the frame is valid, and false if it is not.
277*/
278bool QVideoFrame::isValid() const
279{
280 return d && d->videoBuffer && d->format.pixelFormat() != QVideoFrameFormat::Format_Invalid;
281}
282
283/*!
284 Returns the pixel format of this video frame.
285*/
286QVideoFrameFormat::PixelFormat QVideoFrame::pixelFormat() const
287{
288 return d ? d->format.pixelFormat() : QVideoFrameFormat::Format_Invalid;
289}
290
291/*!
292 Returns the surface format of this video frame.
293*/
294QVideoFrameFormat QVideoFrame::surfaceFormat() const
295{
296 return d ? d->format : QVideoFrameFormat{};
297}
298
299/*!
300 Returns the type of a video frame's handle.
301
302 The handle type could either be NoHandle, meaning that the frame is memory
303 based, or a RHI texture.
304*/
305QVideoFrame::HandleType QVideoFrame::handleType() const
306{
307 return (d && d->hwVideoBuffer) ? d->hwVideoBuffer->handleType() : QVideoFrame::NoHandle;
308}
309
310/*!
311 Returns the dimensions of a video frame.
312*/
313QSize QVideoFrame::size() const
314{
315 return d ? d->format.frameSize() : QSize();
316}
317
318/*!
319 Returns the width of a video frame.
320*/
321int QVideoFrame::width() const
322{
323 return size().width();
324}
325
326/*!
327 Returns the height of a video frame.
328*/
329int QVideoFrame::height() const
330{
331 return size().height();
332}
333
334/*!
335 Identifies if a video frame's contents are currently mapped to system memory.
336
337 This is a convenience function which checks that the \l {QVideoFrame::MapMode}{MapMode}
338 of the frame is not equal to QVideoFrame::NotMapped.
339
340 Returns true if the contents of the video frame are mapped to system memory, and false
341 otherwise.
342
343 \sa mapMode(), QVideoFrame::MapMode
344*/
345
346bool QVideoFrame::isMapped() const
347{
348 return d && d->mapMode != QVideoFrame::NotMapped;
349}
350
351/*!
352 Identifies if the mapped contents of a video frame will be persisted when the frame is unmapped.
353
354 This is a convenience function which checks if the \l {QVideoFrame::MapMode}{MapMode}
355 contains the QVideoFrame::WriteOnly flag.
356
357 Returns true if the video frame will be updated when unmapped, and false otherwise.
358
359 \note The result of altering the data of a frame that is mapped in read-only mode is undefined.
360 Depending on the buffer implementation the changes may be persisted, or worse alter a shared
361 buffer.
362
363 \sa mapMode(), QVideoFrame::MapMode
364*/
365bool QVideoFrame::isWritable() const
366{
367 return d && (d->mapMode & QVideoFrame::WriteOnly);
368}
369
370/*!
371 Identifies if the mapped contents of a video frame were read from the frame when it was mapped.
372
373 This is a convenience function which checks if the \l {QVideoFrame::MapMode}{MapMode}
374 contains the QVideoFrame::WriteOnly flag.
375
376 Returns true if the contents of the mapped memory were read from the video frame, and false
377 otherwise.
378
379 \sa mapMode(), QVideoFrame::MapMode
380*/
381bool QVideoFrame::isReadable() const
382{
383 return d && (d->mapMode & QVideoFrame::ReadOnly);
384}
385
386/*!
387 \enum QVideoFrame::MapMode
388
389 Enumerates how a video buffer's data is mapped to system memory.
390
391 \value NotMapped
392 The video buffer is not mapped to memory.
393 \value ReadOnly
394 The mapped memory is populated with data from the video buffer when mapped,
395 but the content of the mapped memory may be discarded when unmapped.
396 \value WriteOnly
397 The mapped memory is uninitialized when mapped, but the possibly modified
398 content will be used to populate the video buffer when unmapped.
399 \value ReadWrite
400 The mapped memory is populated with data from the video
401 buffer, and the video buffer is repopulated with the content of the mapped
402 memory when it is unmapped.
403
404 \sa mapMode(), map()
405*/
406
407/*!
408 Returns the mode a video frame was mapped to system memory in.
409
410 \sa map(), QVideoFrame::MapMode
411*/
412QVideoFrame::MapMode QVideoFrame::mapMode() const
413{
414 return d ? d->mapMode : QVideoFrame::NotMapped;
415}
416
417/*!
418 Maps the contents of a video frame to system (CPU addressable) memory.
419
420 In some cases the video frame data might be stored in video memory or otherwise inaccessible
421 memory, so it is necessary to map a frame before accessing the pixel data. This may involve
422 copying the contents around, so avoid mapping and unmapping unless required.
423
424 The map \a mode indicates whether the contents of the mapped memory should be read from and/or
425 written to the frame. If the map mode includes the \c QVideoFrame::ReadOnly flag the
426 mapped memory will be populated with the content of the video frame when initially mapped. If the map
427 mode includes the \c QVideoFrame::WriteOnly flag the content of the possibly modified
428 mapped memory will be written back to the frame when unmapped.
429
430 While mapped the contents of a video frame can be accessed directly through the pointer returned
431 by the bits() function.
432
433 When access to the data is no longer needed, be sure to call the unmap() function to release the
434 mapped memory and possibly update the video frame contents.
435
436 If the video frame has been mapped in read only mode, it is permissible to map it
437 multiple times in read only mode (and unmap it a corresponding number of times). In all
438 other cases it is necessary to unmap the frame first before mapping a second time.
439
440 \note Writing to memory that is mapped as read-only is undefined, and may result in changes
441 to shared data or crashes.
442
443 Returns true if the frame was mapped to memory in the given \a mode and false otherwise.
444
445 \sa unmap(), mapMode(), bits()
446*/
447bool QVideoFrame::map(QVideoFrame::MapMode mode)
448{
449 if (!d || !d->videoBuffer)
450 return false;
451
452 QMutexLocker lock(&d->mapMutex);
453 if (mode == QVideoFrame::NotMapped)
454 return false;
455
456 if (d->mappedCount > 0) {
457 //it's allowed to map the video frame multiple times in read only mode
458 if (d->mapMode == QVideoFrame::ReadOnly && mode == QVideoFrame::ReadOnly) {
459 d->mappedCount++;
460 return true;
461 }
462
463 return false;
464 }
465
466 Q_ASSERT(d->mapData.data[0] == nullptr);
467 Q_ASSERT(d->mapData.bytesPerLine[0] == 0);
468 Q_ASSERT(d->mapData.planeCount == 0);
469 Q_ASSERT(d->mapData.dataSize[0] == 0);
470
471 d->mapData = d->videoBuffer->map(mode);
472 if (d->mapData.planeCount == 0)
473 return false;
474
475 d->mapMode = mode;
476
477 if (d->mapData.planeCount == 1) {
478 auto pixelFmt = d->format.pixelFormat();
479 // If the plane count is 1 derive the additional planes for planar formats.
480 switch (pixelFmt) {
481 case QVideoFrameFormat::Format_Invalid:
482 case QVideoFrameFormat::Format_ARGB8888:
483 case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
484 case QVideoFrameFormat::Format_XRGB8888:
485 case QVideoFrameFormat::Format_BGRA8888:
486 case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
487 case QVideoFrameFormat::Format_BGRX8888:
488 case QVideoFrameFormat::Format_ABGR8888:
489 case QVideoFrameFormat::Format_XBGR8888:
490 case QVideoFrameFormat::Format_RGBA8888:
491 case QVideoFrameFormat::Format_RGBX8888:
492 case QVideoFrameFormat::Format_AYUV:
493 case QVideoFrameFormat::Format_AYUV_Premultiplied:
494 case QVideoFrameFormat::Format_UYVY:
495 case QVideoFrameFormat::Format_YUYV:
496 case QVideoFrameFormat::Format_Y8:
497 case QVideoFrameFormat::Format_Y16:
498 case QVideoFrameFormat::Format_Jpeg:
499 case QVideoFrameFormat::Format_SamplerExternalOES:
500 case QVideoFrameFormat::Format_SamplerRect:
501 // Single plane or opaque format.
502 break;
503 case QVideoFrameFormat::Format_YUV420P:
504 case QVideoFrameFormat::Format_YUV420P10:
505 case QVideoFrameFormat::Format_YUV422P:
506 case QVideoFrameFormat::Format_YV12: {
507 // The UV stride is usually half the Y stride and is 32-bit aligned.
508 // However it's not always the case, at least on Windows where the
509 // UV planes are sometimes not aligned.
510 // We calculate the stride using the UV byte count to always
511 // have a correct stride.
512 const int height = this->height();
513 const int yStride = d->mapData.bytesPerLine[0];
514 const int uvHeight = pixelFmt == QVideoFrameFormat::Format_YUV422P ? height : height / 2;
515 const int uvStride = (d->mapData.dataSize[0] - (yStride * height)) / uvHeight / 2;
516
517 // Three planes, the second and third vertically (and horizontally for other than Format_YUV422P formats) subsampled.
518 d->mapData.planeCount = 3;
519 d->mapData.bytesPerLine[2] = d->mapData.bytesPerLine[1] = uvStride;
520 d->mapData.dataSize[0] = yStride * height;
521 d->mapData.dataSize[1] = uvStride * uvHeight;
522 d->mapData.dataSize[2] = uvStride * uvHeight;
523 d->mapData.data[1] = d->mapData.data[0] + d->mapData.dataSize[0];
524 d->mapData.data[2] = d->mapData.data[1] + d->mapData.dataSize[1];
525 break;
526 }
527 case QVideoFrameFormat::Format_NV12:
528 case QVideoFrameFormat::Format_NV21:
529 case QVideoFrameFormat::Format_IMC2:
530 case QVideoFrameFormat::Format_IMC4:
531 case QVideoFrameFormat::Format_P010:
532 case QVideoFrameFormat::Format_P016: {
533 // Semi planar, Full resolution Y plane with interleaved subsampled U and V planes.
534 d->mapData.planeCount = 2;
535 d->mapData.bytesPerLine[1] = d->mapData.bytesPerLine[0];
536 int size = d->mapData.dataSize[0];
537 d->mapData.dataSize[0] = (d->mapData.bytesPerLine[0] * height());
538 d->mapData.dataSize[1] = size - d->mapData.dataSize[0];
539 d->mapData.data[1] = d->mapData.data[0] + d->mapData.dataSize[0];
540 break;
541 }
542 case QVideoFrameFormat::Format_IMC1:
543 case QVideoFrameFormat::Format_IMC3: {
544 // Three planes, the second and third vertically and horizontally subsumpled,
545 // but with lines padded to the width of the first plane.
546 d->mapData.planeCount = 3;
547 d->mapData.bytesPerLine[2] = d->mapData.bytesPerLine[1] = d->mapData.bytesPerLine[0];
548 d->mapData.dataSize[0] = (d->mapData.bytesPerLine[0] * height());
549 d->mapData.dataSize[1] = (d->mapData.bytesPerLine[0] * height() / 2);
550 d->mapData.dataSize[2] = (d->mapData.bytesPerLine[0] * height() / 2);
551 d->mapData.data[1] = d->mapData.data[0] + d->mapData.dataSize[0];
552 d->mapData.data[2] = d->mapData.data[1] + d->mapData.dataSize[1];
553 break;
554 }
555 }
556 }
557
558 d->mappedCount++;
559
560 // unlock mapMutex to avoid potential deadlock imageMutex <--> mapMutex
561 lock.unlock();
562
563 if ((mode & QVideoFrame::WriteOnly) != 0) {
564 QMutexLocker lock(&d->imageMutex);
565 d->image = {};
566 }
567
568 return true;
569}
570
571/*!
572 Releases the memory mapped by the map() function.
573
574 If the \l {QVideoFrame::MapMode}{MapMode} included the QVideoFrame::WriteOnly
575 flag this will persist the current content of the mapped memory to the video frame.
576
577 unmap() should not be called if map() function failed.
578
579 \sa map()
580*/
581void QVideoFrame::unmap()
582{
583 if (!d || !d->videoBuffer)
584 return;
585
586 QMutexLocker lock(&d->mapMutex);
587
588 if (d->mappedCount == 0) {
589 qWarning() << "QVideoFrame::unmap() was called more times then QVideoFrame::map()";
590 return;
591 }
592
593 d->mappedCount--;
594
595 if (d->mappedCount == 0) {
596 d->mapData = {};
597 d->mapMode = QVideoFrame::NotMapped;
598 d->videoBuffer->unmap();
599 }
600}
601
602/*!
603 Returns the number of bytes in a scan line of a \a plane.
604
605 This value is only valid while the frame data is \l {map()}{mapped}.
606
607 \sa bits(), map(), mappedBytes(), planeCount()
608 \since 5.4
609*/
610
611int QVideoFrame::bytesPerLine(int plane) const
612{
613 if (!d)
614 return 0;
615 return plane >= 0 && plane < d->mapData.planeCount ? d->mapData.bytesPerLine[plane] : 0;
616}
617
618/*!
619 Returns a pointer to the start of the frame data buffer for a \a plane.
620
621 This value is only valid while the frame data is \l {map()}{mapped}.
622
623 Changes made to data accessed via this pointer (when mapped with write access)
624 are only guaranteed to have been persisted when unmap() is called and when the
625 buffer has been mapped for writing.
626
627 \sa map(), mappedBytes(), bytesPerLine(), planeCount()
628 \since 5.4
629*/
630uchar *QVideoFrame::bits(int plane)
631{
632 if (!d)
633 return nullptr;
634 return plane >= 0 && plane < d->mapData.planeCount ? d->mapData.data[plane] : nullptr;
635}
636
637/*!
638 Returns a pointer to the start of the frame data buffer for a \a plane.
639
640 This value is only valid while the frame data is \l {map()}{mapped}.
641
642 If the buffer was not mapped with read access, the contents of this
643 buffer will initially be uninitialized.
644
645 \sa map(), mappedBytes(), bytesPerLine(), planeCount()
646 \since 5.4
647*/
648const uchar *QVideoFrame::bits(int plane) const
649{
650 if (!d)
651 return nullptr;
652 return plane >= 0 && plane < d->mapData.planeCount ? d->mapData.data[plane] : nullptr;
653}
654
655/*!
656 Returns the number of bytes occupied by plane \a plane of the mapped frame data.
657
658 This value is only valid while the frame data is \l {map()}{mapped}.
659
660 \sa map()
661*/
662int QVideoFrame::mappedBytes(int plane) const
663{
664 if (!d)
665 return 0;
666 return plane >= 0 && plane < d->mapData.planeCount ? d->mapData.dataSize[plane] : 0;
667}
668
669/*!
670 Returns the number of planes in the video frame.
671
672 \sa map()
673 \since 5.4
674*/
675
676int QVideoFrame::planeCount() const
677{
678 if (!d)
679 return 0;
680 return d->format.planeCount();
681}
682
683/*!
684 Returns the presentation time (in microseconds) when the frame should be displayed.
685
686 An invalid time is represented as -1.
687
688*/
689qint64 QVideoFrame::startTime() const
690{
691 if (!d)
692 return -1;
693 return d->startTime;
694}
695
696/*!
697 Sets the presentation \a time (in microseconds) when the frame should initially be displayed.
698
699 An invalid time is represented as -1.
700
701*/
702void QVideoFrame::setStartTime(qint64 time)
703{
704 if (!d)
705 return;
706 d->startTime = time;
707}
708
709/*!
710 Returns the presentation time (in microseconds) when a frame should stop being displayed.
711
712 An invalid time is represented as -1.
713
714*/
715qint64 QVideoFrame::endTime() const
716{
717 if (!d)
718 return -1;
719 return d->endTime;
720}
721
722/*!
723 Sets the presentation \a time (in microseconds) when a frame should stop being displayed.
724
725 An invalid time is represented as -1.
726
727*/
728void QVideoFrame::setEndTime(qint64 time)
729{
730 if (!d)
731 return;
732 d->endTime = time;
733}
734
735#if QT_DEPRECATED_SINCE(6, 7)
736/*!
737 \enum QVideoFrame::RotationAngle
738 \deprecated [6.7] Use QtVideo::Rotation instead.
739
740 The angle of the clockwise rotation that should be applied to a video
741 frame before displaying.
742
743 \value Rotation0 No rotation required, the frame has correct orientation
744 \value Rotation90 The frame should be rotated by 90 degrees
745 \value Rotation180 The frame should be rotated by 180 degrees
746 \value Rotation270 The frame should be rotated by 270 degrees
747*/
748
749/*!
750 \fn void QVideoFrame::setRotationAngle(RotationAngle)
751 \deprecated [6.7] Use \c QVideoFrame::setRotation instead.
752
753 Sets the \a angle the frame should be rotated clockwise before displaying.
754*/
755
756/*!
757 \fn QVideoFrame::RotationAngle QVideoFrame::rotationAngle() const
758 \deprecated [6.7] Use \c QVideoFrame::rotation instead.
759
760 Returns the angle the frame should be rotated clockwise before displaying.
761*/
762
763#endif
764
765
766/*!
767 Sets the \a angle the frame should be rotated clockwise before displaying.
768
769 Transformations of \c QVideoFrame, specifically rotation and mirroring,
770 are used only for displaying the video frame and are applied on top
771 of the surface transformation, which is determined by \l QVideoFrameFormat.
772 Rotation is applied before mirroring.
773
774 Default value is \c QtVideo::Rotation::None.
775*/
776void QVideoFrame::setRotation(QtVideo::Rotation angle)
777{
778 if (d)
779 d->presentationTransformation.rotation = angle;
780}
781
782/*!
783 Returns the angle the frame should be rotated clockwise before displaying.
784
785 Transformations of \c QVideoFrame, specifically rotation and mirroring,
786 are used only for displaying the video frame and are applied on top
787 of the surface transformation, which is determined by \l QVideoFrameFormat.
788 Rotation is applied before mirroring.
789 */
790QtVideo::Rotation QVideoFrame::rotation() const
791{
792 return d ? d->presentationTransformation.rotation : QtVideo::Rotation::None;
793}
794
795/*!
796 Sets whether the frame should be \a mirrored around its vertical axis before displaying.
797
798 Transformations of \c QVideoFrame, specifically rotation and mirroring,
799 are used only for displaying the video frame and are applied on top
800 of the surface transformation, which is determined by \l QVideoFrameFormat.
801 Mirroring is applied after rotation.
802
803 Mirroring is typically needed for video frames coming from a front camera of a mobile device.
804
805 Default value is \c false.
806*/
807void QVideoFrame::setMirrored(bool mirrored)
808{
809 if (d)
810 d->presentationTransformation.mirrorredHorizontallyAfterRotation = mirrored;
811}
812
813/*!
814 Returns whether the frame should be mirrored around its vertical axis before displaying.
815
816 Transformations of \c QVideoFrame, specifically rotation and mirroring,
817 are used only for displaying the video frame and are applied on top
818 of the surface transformation, which is determined by \l QVideoFrameFormat.
819 Mirroring is applied after rotation.
820
821 Mirroring is typically needed for video frames coming from a front camera of a mobile device.
822*/
823bool QVideoFrame::mirrored() const
824{
825 return d && d->presentationTransformation.mirrorredHorizontallyAfterRotation;
826}
827
828/*!
829 Sets the frame \a rate of a video stream in frames per second.
830*/
831void QVideoFrame::setStreamFrameRate(qreal rate)
832{
833 if (d)
834 d->format.setStreamFrameRate(rate);
835}
836
837/*!
838 Returns the frame rate of a video stream in frames per second.
839*/
840qreal QVideoFrame::streamFrameRate() const
841{
842 return d ? d->format.streamFrameRate() : 0.;
843}
844
845/*!
846 Converts current video frame to image.
847
848 The consversion is based on the current pixel data and
849 the \l {surface format}{QVideoFrame::surfaceFormat}.
850 Transformations of the frame don't impact the result
851 since they are applied for presentation only.
852
853 \since 5.15
854*/
855QImage QVideoFrame::toImage() const
856{
857 if (!isValid())
858 return {};
859
860 QMutexLocker lock(&d->imageMutex);
861
862 if (d->image.isNull())
863 d->image = qImageFromVideoFrame(frame: *this, transformation: qNormalizedSurfaceTransformation(format: d->format));
864
865 return d->image;
866}
867
868/*!
869 Returns the subtitle text that should be rendered together with this video frame.
870*/
871QString QVideoFrame::subtitleText() const
872{
873 return d ? d->subtitleText : QString();
874}
875
876/*!
877 Sets the subtitle text that should be rendered together with this video frame to \a text.
878*/
879void QVideoFrame::setSubtitleText(const QString &text)
880{
881 if (!d)
882 return;
883 d->subtitleText = text;
884}
885
886/*!
887 Uses a QPainter, \a{painter}, to render this QVideoFrame to \a rect.
888 The PaintOptions \a options can be used to specify a background color and
889 how \a rect should be filled with the video.
890
891 \note that rendering will usually happen without hardware acceleration when
892 using this method.
893*/
894void QVideoFrame::paint(QPainter *painter, const QRectF &rect, const PaintOptions &options)
895{
896 if (!isValid()) {
897 painter->fillRect(rect, color: options.backgroundColor);
898 return;
899 }
900
901 QRectF targetRect = rect;
902 QSizeF size = qRotatedFramePresentationSize(frame: *this);
903
904 size.scale(s: targetRect.size(), mode: options.aspectRatioMode);
905
906 if (options.aspectRatioMode == Qt::KeepAspectRatio) {
907 targetRect = QRect(0, 0, size.width(), size.height());
908 targetRect.moveCenter(p: rect.center());
909 // we might not be drawing every pixel, fill the leftovers black
910 if (options.backgroundColor != Qt::transparent && rect != targetRect) {
911 if (targetRect.top() > rect.top()) {
912 QRectF top(rect.left(), rect.top(), rect.width(), targetRect.top() - rect.top());
913 painter->fillRect(r: top, c: Qt::black);
914 }
915 if (targetRect.left() > rect.left()) {
916 QRectF top(rect.left(), targetRect.top(), targetRect.left() - rect.left(), targetRect.height());
917 painter->fillRect(r: top, c: Qt::black);
918 }
919 if (targetRect.right() < rect.right()) {
920 QRectF top(targetRect.right(), targetRect.top(), rect.right() - targetRect.right(), targetRect.height());
921 painter->fillRect(r: top, c: Qt::black);
922 }
923 if (targetRect.bottom() < rect.bottom()) {
924 QRectF top(rect.left(), targetRect.bottom(), rect.width(), rect.bottom() - targetRect.bottom());
925 painter->fillRect(r: top, c: Qt::black);
926 }
927 }
928 }
929
930 if (map(mode: QVideoFrame::ReadOnly)) {
931 const QTransform oldTransform = painter->transform();
932 QTransform transform = oldTransform;
933 transform.translate(dx: targetRect.center().x() - size.width()/2,
934 dy: targetRect.center().y() - size.height()/2);
935 painter->setTransform(transform);
936
937 const bool hasPresentationTransformation =
938 d->presentationTransformation != VideoTransformation{};
939
940 // Use cache for images without presentation transform
941 const QImage image = hasPresentationTransformation
942 ? qImageFromVideoFrame(frame: *this, transformation: qNormalizedFrameTransformation(frame: *this))
943 : toImage();
944
945 painter->drawImage(targetRect: {{}, size}, image, sourceRect: {{},image.size()});
946 painter->setTransform(transform: oldTransform);
947
948 unmap();
949 } else if (isValid()) {
950 // #### error handling
951 } else {
952 painter->fillRect(r: rect, c: Qt::black);
953 }
954
955 if ((options.paintFlags & PaintOptions::DontDrawSubtitles) || d->subtitleText.isEmpty())
956 return;
957
958 // draw subtitles
959 auto text = d->subtitleText;
960 text.replace(before: QLatin1Char('\n'), after: QChar::LineSeparator);
961
962 QVideoTextureHelper::SubtitleLayout layout;
963 layout.update(frameSize: targetRect.size().toSize(), text: this->subtitleText());
964 layout.draw(painter, translate: targetRect.topLeft());
965}
966
967#ifndef QT_NO_DEBUG_STREAM
968static QString qFormatTimeStamps(qint64 start, qint64 end)
969{
970 // Early out for invalid.
971 if (start < 0)
972 return QLatin1String("[no timestamp]");
973
974 bool onlyOne = (start == end);
975
976 // [hh:]mm:ss.ms
977 const int s_millis = start % 1000000;
978 start /= 1000000;
979 const int s_seconds = start % 60;
980 start /= 60;
981 const int s_minutes = start % 60;
982 start /= 60;
983
984 if (onlyOne) {
985 if (start > 0)
986 return QStringLiteral("@%1:%2:%3.%4")
987 .arg(a: start, fieldwidth: 1, base: 10, fillChar: QLatin1Char('0'))
988 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
989 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
990 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
991 return QStringLiteral("@%1:%2.%3")
992 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
993 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
994 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
995 }
996
997 if (end == -1) {
998 // Similar to start-start, except it means keep displaying it?
999 if (start > 0)
1000 return QStringLiteral("%1:%2:%3.%4 - forever")
1001 .arg(a: start, fieldwidth: 1, base: 10, fillChar: QLatin1Char('0'))
1002 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1003 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1004 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
1005 return QStringLiteral("%1:%2.%3 - forever")
1006 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1007 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1008 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
1009 }
1010
1011 const int e_millis = end % 1000000;
1012 end /= 1000000;
1013 const int e_seconds = end % 60;
1014 end /= 60;
1015 const int e_minutes = end % 60;
1016 end /= 60;
1017
1018 if (start > 0 || end > 0)
1019 return QStringLiteral("%1:%2:%3.%4 - %5:%6:%7.%8")
1020 .arg(a: start, fieldwidth: 1, base: 10, fillChar: QLatin1Char('0'))
1021 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1022 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1023 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1024 .arg(a: end, fieldwidth: 1, base: 10, fillChar: QLatin1Char('0'))
1025 .arg(a: e_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1026 .arg(a: e_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1027 .arg(a: e_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
1028 return QStringLiteral("%1:%2.%3 - %4:%5.%6")
1029 .arg(a: s_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1030 .arg(a: s_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1031 .arg(a: s_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1032 .arg(a: e_minutes, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1033 .arg(a: e_seconds, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'))
1034 .arg(a: e_millis, fieldWidth: 2, base: 10, fillChar: QLatin1Char('0'));
1035}
1036
1037QDebug operator<<(QDebug dbg, QVideoFrame::HandleType type)
1038{
1039 QDebugStateSaver saver(dbg);
1040 dbg.nospace();
1041 switch (type) {
1042 case QVideoFrame::NoHandle:
1043 return dbg << "NoHandle";
1044 case QVideoFrame::RhiTextureHandle:
1045 return dbg << "RhiTextureHandle";
1046 }
1047 return dbg;
1048}
1049
1050QDebug operator<<(QDebug dbg, const QVideoFrame& f)
1051{
1052 QDebugStateSaver saver(dbg);
1053 dbg.nospace();
1054 dbg << "QVideoFrame(" << f.size() << ", "
1055 << f.pixelFormat() << ", "
1056 << f.handleType() << ", "
1057 << f.mapMode() << ", "
1058 << qFormatTimeStamps(start: f.startTime(), end: f.endTime()).toLatin1().constData();
1059 dbg << ')';
1060 return dbg;
1061}
1062#endif
1063
1064QT_END_NAMESPACE
1065
1066

source code of qtmultimedia/src/multimedia/video/qvideoframe.cpp