1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpegconverter_p.h"
5#include "qffmpeg_p.h"
6#include <QtMultimedia/qvideoframeformat.h>
7#include <QtMultimedia/qvideoframe.h>
8#include <QtCore/qloggingcategory.h>
9#include <private/qvideotexturehelper_p.h>
10
11extern "C" {
12#include <libswscale/swscale.h>
13}
14
15QT_BEGIN_NAMESPACE
16
17namespace {
18
19Q_LOGGING_CATEGORY(lc, "qt.multimedia.ffmpeg.converter");
20
21
22// Converts to FFmpeg pixel format. This function differs from
23// QFFmpegVideoBuffer::toAVPixelFormat which only covers the subset
24// of pixel formats required for encoding. Here we need to cover more
25// pixel formats to be able to generate test images for decoding/display
26AVPixelFormat toAVPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat)
27{
28 switch (pixelFormat) {
29 default:
30 case QVideoFrameFormat::Format_Invalid:
31 return AV_PIX_FMT_NONE;
32 case QVideoFrameFormat::Format_AYUV:
33 case QVideoFrameFormat::Format_AYUV_Premultiplied:
34 return AV_PIX_FMT_NONE; // TODO: Fixme (No corresponding FFmpeg format available)
35 case QVideoFrameFormat::Format_YV12:
36 case QVideoFrameFormat::Format_IMC1:
37 case QVideoFrameFormat::Format_IMC3:
38 case QVideoFrameFormat::Format_IMC2:
39 case QVideoFrameFormat::Format_IMC4:
40 return AV_PIX_FMT_YUV420P;
41 case QVideoFrameFormat::Format_Jpeg:
42 return AV_PIX_FMT_BGRA;
43 case QVideoFrameFormat::Format_ARGB8888:
44 return AV_PIX_FMT_ARGB;
45 case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
46 case QVideoFrameFormat::Format_XRGB8888:
47 return AV_PIX_FMT_0RGB;
48 case QVideoFrameFormat::Format_BGRA8888:
49 return AV_PIX_FMT_BGRA;
50 case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
51 case QVideoFrameFormat::Format_BGRX8888:
52 return AV_PIX_FMT_BGR0;
53 case QVideoFrameFormat::Format_ABGR8888:
54 return AV_PIX_FMT_ABGR;
55 case QVideoFrameFormat::Format_XBGR8888:
56 return AV_PIX_FMT_0BGR;
57 case QVideoFrameFormat::Format_RGBA8888:
58 return AV_PIX_FMT_RGBA;
59 case QVideoFrameFormat::Format_RGBX8888:
60 return AV_PIX_FMT_RGB0;
61 case QVideoFrameFormat::Format_YUV422P:
62 return AV_PIX_FMT_YUV422P;
63 case QVideoFrameFormat::Format_YUV420P:
64 return AV_PIX_FMT_YUV420P;
65 case QVideoFrameFormat::Format_YUV420P10:
66 return AV_PIX_FMT_YUV420P10;
67 case QVideoFrameFormat::Format_UYVY:
68 return AV_PIX_FMT_UYVY422;
69 case QVideoFrameFormat::Format_YUYV:
70 return AV_PIX_FMT_YUYV422;
71 case QVideoFrameFormat::Format_NV12:
72 return AV_PIX_FMT_NV12;
73 case QVideoFrameFormat::Format_NV21:
74 return AV_PIX_FMT_NV21;
75 case QVideoFrameFormat::Format_Y8:
76 return AV_PIX_FMT_GRAY8;
77 case QVideoFrameFormat::Format_Y16:
78 return AV_PIX_FMT_GRAY16;
79 case QVideoFrameFormat::Format_P010:
80 return AV_PIX_FMT_P010;
81 case QVideoFrameFormat::Format_P016:
82 return AV_PIX_FMT_P016;
83 case QVideoFrameFormat::Format_SamplerExternalOES:
84 return AV_PIX_FMT_MEDIACODEC;
85 }
86}
87
88struct SwsFrameData
89{
90 static constexpr int arraySize = 4; // Array size required by sws_scale
91 std::array<uchar *, arraySize> bits;
92 std::array<int, arraySize> stride;
93};
94
95SwsFrameData getSwsData(QVideoFrame &dst)
96{
97 switch (dst.pixelFormat()) {
98 case QVideoFrameFormat::Format_YV12:
99 case QVideoFrameFormat::Format_IMC1:
100 return { .bits: { dst.bits(plane: 0), dst.bits(plane: 2), dst.bits(plane: 1), nullptr },
101 .stride: { dst.bytesPerLine(plane: 0), dst.bytesPerLine(plane: 2), dst.bytesPerLine(plane: 1), 0 } };
102
103 case QVideoFrameFormat::Format_IMC2:
104 return { .bits: { dst.bits(plane: 0), dst.bits(plane: 1) + dst.bytesPerLine(plane: 1) / 2, dst.bits(plane: 1), nullptr },
105 .stride: { dst.bytesPerLine(plane: 0), dst.bytesPerLine(plane: 1), dst.bytesPerLine(plane: 1), 0 } };
106
107 case QVideoFrameFormat::Format_IMC4:
108 return { .bits: { dst.bits(plane: 0), dst.bits(plane: 1), dst.bits(plane: 1) + dst.bytesPerLine(plane: 1) / 2, nullptr },
109 .stride: { dst.bytesPerLine(plane: 0), dst.bytesPerLine(plane: 1), dst.bytesPerLine(plane: 1), 0 } };
110 default:
111 return { .bits: { dst.bits(plane: 0), dst.bits(plane: 1), dst.bits(plane: 2), nullptr },
112 .stride: { dst.bytesPerLine(plane: 0), dst.bytesPerLine(plane: 1), dst.bytesPerLine(plane: 2), 0 } };
113 }
114}
115
116struct SwsColorSpace
117{
118 int colorSpace;
119 int colorRange; // 0 - mpeg/video, 1 - jpeg/full
120};
121
122// Qt heuristics for determining color space requires checking
123// both frame color space and range. This function mimics logic
124// used elsewhere in Qt Multimedia.
125SwsColorSpace toSwsColorSpace(QVideoFrameFormat::ColorRange colorRange,
126 QVideoFrameFormat::ColorSpace colorSpace)
127{
128 const int avRange = colorRange == QVideoFrameFormat::ColorRange_Video ? 0 : 1;
129
130 switch (colorSpace) {
131 case QVideoFrameFormat::ColorSpace_BT601:
132 if (colorRange == QVideoFrameFormat::ColorRange_Full)
133 return { SWS_CS_ITU709, .colorRange: 1 }; // TODO: FIXME - Not exact match
134 return { SWS_CS_ITU601, .colorRange: 0 };
135 case QVideoFrameFormat::ColorSpace_BT709:
136 return { SWS_CS_ITU709, .colorRange: avRange };
137 case QVideoFrameFormat::ColorSpace_AdobeRgb:
138 return { SWS_CS_ITU601, .colorRange: 1 }; // TODO: Why do ITU601 and Adobe RGB match well?
139 case QVideoFrameFormat::ColorSpace_BT2020:
140 return { SWS_CS_BT2020, .colorRange: avRange };
141 case QVideoFrameFormat::ColorSpace_Undefined:
142 default:
143 return { SWS_CS_DEFAULT, .colorRange: avRange };
144 }
145}
146
147using PixelFormat = QVideoFrameFormat::PixelFormat;
148
149// clang-format off
150
151QFFmpeg::SwsContextUPtr createConverter(const QSize &srcSize, PixelFormat srcPixFmt,
152 const QSize &dstSize, PixelFormat dstPixFmt)
153{
154 return QFFmpeg::createSwsContext(srcSize, srcPixFmt: toAVPixelFormat(pixelFormat: srcPixFmt), dstSize, dstPixFmt: toAVPixelFormat(pixelFormat: dstPixFmt), SWS_BILINEAR);
155}
156
157bool setColorSpaceDetails(SwsContext *context,
158 const QVideoFrameFormat &srcFormat,
159 const QVideoFrameFormat &dstFormat)
160{
161 const SwsColorSpace src = toSwsColorSpace(colorRange: srcFormat.colorRange(), colorSpace: srcFormat.colorSpace());
162 const SwsColorSpace dst = toSwsColorSpace(colorRange: dstFormat.colorRange(), colorSpace: dstFormat.colorSpace());
163
164 constexpr int brightness = 0;
165 constexpr int contrast = 0;
166 constexpr int saturation = 0;
167 const int status = sws_setColorspaceDetails(c: context,
168 inv_table: sws_getCoefficients(colorspace: src.colorSpace), srcRange: src.colorRange,
169 table: sws_getCoefficients(colorspace: dst.colorSpace), dstRange: dst.colorRange,
170 brightness, contrast, saturation);
171
172 return status == 0;
173}
174
175bool convert(SwsContext *context, QVideoFrame &src, int srcHeight, QVideoFrame &dst)
176{
177 if (!src.map(mode: QVideoFrame::ReadOnly))
178 return false;
179
180 QScopeGuard unmapSrc{[&] {
181 src.unmap();
182 }};
183
184 if (!dst.map(mode: QVideoFrame::WriteOnly))
185 return false;
186
187 QScopeGuard unmapDst{[&] {
188 dst.unmap();
189 }};
190
191 const SwsFrameData srcData = getSwsData(dst&: src);
192 const SwsFrameData dstData = getSwsData(dst);
193
194 constexpr int firstSrcSliceRow = 0;
195 const int scaledHeight = sws_scale(c: context,
196 srcSlice: srcData.bits.data(), srcStride: srcData.stride.data(),
197 srcSliceY: firstSrcSliceRow, srcSliceH: srcHeight,
198 dst: dstData.bits.data(), dstStride: dstData.stride.data());
199
200 if (scaledHeight != srcHeight)
201 return false;
202
203 return true;
204}
205
206// Ensure even size if using planar format with chroma subsampling
207QSize adjustSize(const QSize& size, PixelFormat srcFmt, PixelFormat dstFmt)
208{
209 const auto* srcDesc = QVideoTextureHelper::textureDescription(format: srcFmt);
210 const auto* dstDesc = QVideoTextureHelper::textureDescription(format: dstFmt);
211
212 QSize output = size;
213 for (const auto desc : { srcDesc, dstDesc }) {
214 for (int i = 0; i < desc->nplanes; ++i) {
215 // TODO: Assumes that max subsampling is 2
216 if (desc->sizeScale[i].x != 1)
217 output.setWidth(output.width() & ~1); // Make even
218
219 if (desc->sizeScale[i].y != 1)
220 output.setHeight(output.height() & ~1); // Make even
221 }
222 }
223
224 return output;
225}
226
227} // namespace
228
229// Converts a video frame to the dstFormat video frame format.
230QVideoFrame convertFrame(QVideoFrame &src, const QVideoFrameFormat &dstFormat)
231{
232 if (src.size() != dstFormat.frameSize()) {
233 qCCritical(lc) << "Resizing is not supported";
234 return {};
235 }
236
237 // Adjust size to even width/height if we have chroma subsampling
238 const QSize size = adjustSize(size: src.size(), srcFmt: src.pixelFormat(), dstFmt: dstFormat.pixelFormat());
239 if (size != src.size())
240 qCWarning(lc) << "Input truncated to even width/height";
241
242 const QFFmpeg::SwsContextUPtr conv = createConverter(
243 srcSize: size, srcPixFmt: src.pixelFormat(), dstSize: size, dstPixFmt: dstFormat.pixelFormat());
244
245 if (!conv) {
246 qCCritical(lc) << "Failed to create SW converter";
247 return {};
248 }
249
250 if (!setColorSpaceDetails(context: conv.get(), srcFormat: src.surfaceFormat(), dstFormat)) {
251 qCCritical(lc) << "Failed to set color space details";
252 return {};
253 }
254
255 QVideoFrame dst{ dstFormat };
256
257 if (!convert(context: conv.get(), src, srcHeight: size.height(), dst)) {
258 qCCritical(lc) << "Frame conversion failed";
259 return {};
260 }
261
262 return dst;
263}
264
265// clang-format on
266
267QT_END_NAMESPACE
268

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpegconverter.cpp