1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpegcodecstorage_p.h"
5
6#include "qffmpeg_p.h"
7#include "qffmpeghwaccel_p.h"
8
9#include <qdebug.h>
10#include <qloggingcategory.h>
11
12#include <algorithm>
13#include <vector>
14#include <array>
15
16#include <unordered_set>
17
18extern "C" {
19#include <libavutil/pixdesc.h>
20#include <libavutil/samplefmt.h>
21}
22
23#ifdef Q_OS_ANDROID
24# include <QtCore/qjniobject.h>
25# include <QtCore/qjniarray.h>
26# include <QtCore/qjnitypes.h>
27#endif
28
29QT_BEGIN_NAMESPACE
30
31#ifdef Q_OS_ANDROID
32Q_DECLARE_JNI_CLASS(QtVideoDeviceManager,
33 "org/qtproject/qt/android/multimedia/QtVideoDeviceManager")
34#endif // Q_OS_ANDROID
35
36static Q_LOGGING_CATEGORY(qLcCodecStorage, "qt.multimedia.ffmpeg.codecstorage");
37
38namespace QFFmpeg {
39
40namespace {
41
42enum CodecStorageType {
43 Encoders,
44 Decoders,
45
46 // TODO: maybe split sw/hw codecs
47
48 CodecStorageTypeCount
49};
50
51using CodecsStorage = std::vector<Codec>;
52
53struct CodecsComparator
54{
55 bool operator()(const Codec &a, const Codec &b) const
56 {
57 return a.id() < b.id() || (a.id() == b.id() && a.isExperimental() < b.isExperimental());
58 }
59
60 bool operator()(const Codec &codec, AVCodecID id) const { return codec.id() < id; }
61 bool operator()(AVCodecID id, const Codec &codec) const { return id < codec.id(); }
62};
63
64template <typename FlagNames>
65QString flagsToString(int flags, const FlagNames &flagNames)
66{
67 QString result;
68 int leftover = flags;
69 for (const auto &flagAndName : flagNames)
70 if ((flags & flagAndName.first) != 0) {
71 leftover &= ~flagAndName.first;
72 if (!result.isEmpty())
73 result += u", ";
74 result += QLatin1StringView(flagAndName.second);
75 }
76
77 if (leftover) {
78 if (!result.isEmpty())
79 result += u", ";
80 result += QString::number(leftover, base: 16);
81 }
82 return result;
83}
84
85void dumpCodecInfo(const Codec &codec)
86{
87 using FlagNames = std::initializer_list<std::pair<int, const char *>>;
88 const auto mediaType = codec.type() == AVMEDIA_TYPE_VIDEO ? "video"
89 : codec.type() == AVMEDIA_TYPE_AUDIO ? "audio"
90 : codec.type() == AVMEDIA_TYPE_SUBTITLE ? "subtitle"
91 : "other_type";
92
93 const auto type = codec.isEncoder()
94 ? codec.isDecoder() ? "encoder/decoder:" : "encoder:"
95 : "decoder:";
96
97 static const FlagNames capabilitiesNames = {
98 { AV_CODEC_CAP_DRAW_HORIZ_BAND, "DRAW_HORIZ_BAND" },
99 { AV_CODEC_CAP_DR1, "DRAW_HORIZ_DR1" },
100 { AV_CODEC_CAP_DELAY, "DELAY" },
101 { AV_CODEC_CAP_SMALL_LAST_FRAME, "SMALL_LAST_FRAME" },
102 { AV_CODEC_CAP_SUBFRAMES, "SUBFRAMES" },
103 { AV_CODEC_CAP_EXPERIMENTAL, "EXPERIMENTAL" },
104 { AV_CODEC_CAP_CHANNEL_CONF, "CHANNEL_CONF" },
105 { AV_CODEC_CAP_FRAME_THREADS, "FRAME_THREADS" },
106 { AV_CODEC_CAP_SLICE_THREADS, "SLICE_THREADS" },
107 { AV_CODEC_CAP_PARAM_CHANGE, "PARAM_CHANGE" },
108#ifdef AV_CODEC_CAP_OTHER_THREADS
109 { AV_CODEC_CAP_OTHER_THREADS, "OTHER_THREADS" },
110#endif
111 { AV_CODEC_CAP_VARIABLE_FRAME_SIZE, "VARIABLE_FRAME_SIZE" },
112 { AV_CODEC_CAP_AVOID_PROBING, "AVOID_PROBING" },
113 { AV_CODEC_CAP_HARDWARE, "HARDWARE" },
114 { AV_CODEC_CAP_HYBRID, "HYBRID" },
115 { AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE, "ENCODER_REORDERED_OPAQUE" },
116#ifdef AV_CODEC_CAP_ENCODER_FLUSH
117 { AV_CODEC_CAP_ENCODER_FLUSH, "ENCODER_FLUSH" },
118#endif
119 };
120
121 qCDebug(qLcCodecStorage) << mediaType << type << codec.name() << "id:" << codec.id()
122 << "capabilities:"
123 << flagsToString(flags: codec.capabilities(), flagNames: capabilitiesNames);
124
125 if (codec.type() == AVMEDIA_TYPE_VIDEO) {
126 const auto pixelFormats = codec.pixelFormats();
127 if (!pixelFormats.empty()) {
128 static const FlagNames flagNames = {
129 { AV_PIX_FMT_FLAG_BE, "BE" },
130 { AV_PIX_FMT_FLAG_PAL, "PAL" },
131 { AV_PIX_FMT_FLAG_BITSTREAM, "BITSTREAM" },
132 { AV_PIX_FMT_FLAG_HWACCEL, "HWACCEL" },
133 { AV_PIX_FMT_FLAG_PLANAR, "PLANAR" },
134 { AV_PIX_FMT_FLAG_RGB, "RGB" },
135 { AV_PIX_FMT_FLAG_ALPHA, "ALPHA" },
136 { AV_PIX_FMT_FLAG_BAYER, "BAYER" },
137 { AV_PIX_FMT_FLAG_FLOAT, "FLOAT" },
138 };
139
140 qCDebug(qLcCodecStorage) << " pixelFormats:";
141 for (AVPixelFormat f : pixelFormats) {
142 auto desc = av_pix_fmt_desc_get(pix_fmt: f);
143 qCDebug(qLcCodecStorage)
144 << " id:" << f << desc->name << "depth:" << desc->comp[0].depth
145 << "flags:" << flagsToString(flags: desc->flags, flagNames);
146 }
147 } else {
148 qCDebug(qLcCodecStorage) << " pixelFormats: null";
149 }
150 } else if (codec.type() == AVMEDIA_TYPE_AUDIO) {
151 const auto sampleFormats = codec.sampleFormats();
152 if (!sampleFormats.empty()) {
153 qCDebug(qLcCodecStorage) << " sampleFormats:";
154 for (auto f : sampleFormats) {
155 const auto name = av_get_sample_fmt_name(sample_fmt: f);
156 qCDebug(qLcCodecStorage) << " id:" << f << (name ? name : "unknown")
157 << "bytes_per_sample:" << av_get_bytes_per_sample(sample_fmt: f)
158 << "is_planar:" << av_sample_fmt_is_planar(sample_fmt: f);
159 }
160 } else {
161 qCDebug(qLcCodecStorage) << " sampleFormats: null";
162 }
163 }
164
165 const std::vector<const AVCodecHWConfig*> hwConfigs = codec.hwConfigs();
166 if (!hwConfigs.empty()) {
167 static const FlagNames hwConfigMethodNames = {
168 { AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, "HW_DEVICE_CTX" },
169 { AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX, "HW_FRAMES_CTX" },
170 { AV_CODEC_HW_CONFIG_METHOD_INTERNAL, "INTERNAL" },
171 { AV_CODEC_HW_CONFIG_METHOD_AD_HOC, "AD_HOC" }
172 };
173
174 qCDebug(qLcCodecStorage) << " hw config:";
175 for (const AVCodecHWConfig* config : hwConfigs) {
176 const auto pixFmtForDevice = pixelFormatForHwDevice(deviceType: config->device_type);
177 auto pixFmtDesc = av_pix_fmt_desc_get(pix_fmt: config->pix_fmt);
178 auto pixFmtForDeviceDesc = av_pix_fmt_desc_get(pix_fmt: pixFmtForDevice);
179 qCDebug(qLcCodecStorage)
180 << " device_type:" << config->device_type << "pix_fmt:" << config->pix_fmt
181 << (pixFmtDesc ? pixFmtDesc->name : "unknown")
182 << "pixelFormatForHwDevice:" << pixelFormatForHwDevice(deviceType: config->device_type)
183 << (pixFmtForDeviceDesc ? pixFmtForDeviceDesc->name : "unknown")
184 << "hw_config_methods:" << flagsToString(flags: config->methods, flagNames: hwConfigMethodNames);
185 }
186 }
187}
188
189bool isCodecValid(const Codec &codec, const std::vector<AVHWDeviceType> &availableHwDeviceTypes,
190 const std::optional<std::unordered_set<AVCodecID>> &codecAvailableOnDevice)
191{
192 if (codec.type() != AVMEDIA_TYPE_VIDEO)
193 return true;
194
195 const auto pixelFormats = codec.pixelFormats();
196 if (pixelFormats.empty()) {
197#if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
198 // Disable V4L2 M2M codecs for encoding for now,
199 // TODO: Investigate on how to get them working
200 if (codec.name().contains(s: QLatin1StringView{ "_v4l2m2m" }) && codec.isEncoder())
201 return false;
202
203 // MediaCodec in Android is used for hardware-accelerated media processing. That is why
204 // before marking it as valid, we need to make sure if it is available on current device.
205 if (codec.name().contains(s: QLatin1StringView{ "_mediacodec" })
206 && (codec.capabilities() & AV_CODEC_CAP_HARDWARE)
207 && codecAvailableOnDevice && codecAvailableOnDevice->count(x: codec.id()) == 0)
208 return false;
209#endif
210
211 return true; // When the codec reports no pixel formats, format support is unknown.
212 }
213
214 if (!findAVPixelFormat(codec, predicate: &isHwPixelFormat))
215 return true; // Codec does not support any hw pixel formats, so no further checks are needed
216
217 if ((codec.capabilities() & AV_CODEC_CAP_HARDWARE) == 0)
218 return true; // Codec does not support hardware processing, so no further checks are needed
219
220 if (codecAvailableOnDevice && codecAvailableOnDevice->count(x: codec.id()) == 0)
221 return false; // Codec is not in platform's allow-list
222
223 auto checkDeviceType = [codec](AVHWDeviceType type) {
224 return isAVFormatSupported(codec, format: pixelFormatForHwDevice(deviceType: type));
225 };
226
227 return std::any_of(first: availableHwDeviceTypes.begin(), last: availableHwDeviceTypes.end(),
228 pred: checkDeviceType);
229}
230
231std::optional<std::unordered_set<AVCodecID>> availableHWCodecs(const CodecStorageType type)
232{
233#ifdef Q_OS_ANDROID
234 using namespace Qt::StringLiterals;
235 using namespace QtJniTypes;
236 std::unordered_set<AVCodecID> availabeCodecs;
237
238 auto getCodecId = [](const QString &codecName) {
239 if (codecName == "3gpp"_L1)
240 return AV_CODEC_ID_H263;
241 if (codecName == "avc"_L1)
242 return AV_CODEC_ID_H264;
243 if (codecName == "hevc"_L1)
244 return AV_CODEC_ID_HEVC;
245 if (codecName == "mp4v-es"_L1)
246 return AV_CODEC_ID_MPEG4;
247 if (codecName == "x-vnd.on2.vp8"_L1)
248 return AV_CODEC_ID_VP8;
249 if (codecName == "x-vnd.on2.vp9"_L1)
250 return AV_CODEC_ID_VP9;
251 return AV_CODEC_ID_NONE;
252 };
253
254 const QJniArray jniCodecs = QtVideoDeviceManager::callStaticMethod<String[]>(
255 type == Encoders ? "getHWVideoEncoders" : "getHWVideoDecoders");
256
257 for (const auto &codec : jniCodecs)
258 availabeCodecs.insert(getCodecId(codec.toString()));
259 return availabeCodecs;
260#else
261 Q_UNUSED(type);
262 return {};
263#endif
264}
265
266const CodecsStorage &codecsStorage(CodecStorageType codecsType)
267{
268 static const auto &storages = []() {
269 std::array<CodecsStorage, CodecStorageTypeCount> result;
270 const auto platformHwEncoders = availableHWCodecs(type: Encoders);
271 const auto platformHwDecoders = availableHWCodecs(type: Decoders);
272
273 for (const Codec codec : CodecEnumerator()) {
274 // TODO: to be investigated
275 // FFmpeg functions avcodec_find_decoder/avcodec_find_encoder
276 // find experimental codecs in the last order,
277 // now we don't consider them at all since they are supposed to
278 // be not stable, maybe we shouldn't.
279 // Currently, it's possible to turn them on for testing purposes.
280
281 static const auto experimentalCodecsEnabled =
282 qEnvironmentVariableIntValue(varName: "QT_ENABLE_EXPERIMENTAL_CODECS");
283
284 if (!experimentalCodecsEnabled && codec.isExperimental()) {
285 qCDebug(qLcCodecStorage) << "Skip experimental codec" << codec.name();
286 continue;
287 }
288
289 if (codec.isDecoder()) {
290 if (isCodecValid(codec, availableHwDeviceTypes: HWAccel::decodingDeviceTypes(), codecAvailableOnDevice: platformHwDecoders))
291 result[Decoders].emplace_back(args: codec);
292 else
293 qCDebug(qLcCodecStorage)
294 << "Skip decoder" << codec.name()
295 << "due to disabled matching hw acceleration, or dysfunctional codec";
296 }
297
298 if (codec.isEncoder()) {
299 if (isCodecValid(codec, availableHwDeviceTypes: HWAccel::encodingDeviceTypes(), codecAvailableOnDevice: platformHwEncoders))
300 result[Encoders].emplace_back(args: codec);
301 else
302 qCDebug(qLcCodecStorage)
303 << "Skip encoder" << codec.name()
304 << "due to disabled matching hw acceleration, or dysfunctional codec";
305 }
306 }
307
308 for (auto &storage : result) {
309 storage.shrink_to_fit();
310
311 // we should ensure the original order
312 std::stable_sort(first: storage.begin(), last: storage.end(), comp: CodecsComparator{});
313 }
314
315 // It print pretty much logs, so let's print it only for special case
316 const bool shouldDumpCodecsInfo = qLcCodecStorage().isEnabled(type: QtDebugMsg)
317 && qEnvironmentVariableIsSet(varName: "QT_FFMPEG_DEBUG");
318
319 if (shouldDumpCodecsInfo) {
320 qCDebug(qLcCodecStorage) << "Advanced FFmpeg codecs info:";
321 for (auto &storage : result) {
322 std::for_each(first: storage.begin(), last: storage.end(), f: &dumpCodecInfo);
323 qCDebug(qLcCodecStorage) << "---------------------------";
324 }
325 }
326
327 return result;
328 }();
329
330 return storages[codecsType];
331}
332
333template <typename CodecScoreGetter, typename CodecOpener>
334bool findAndOpenCodec(CodecStorageType codecsType, AVCodecID codecId,
335 const CodecScoreGetter &scoreGetter, const CodecOpener &opener)
336{
337 Q_ASSERT(opener);
338 const auto &storage = codecsStorage(codecsType);
339 auto it = std::lower_bound(first: storage.begin(), last: storage.end(), val: codecId, comp: CodecsComparator{});
340
341 using CodecToScore = std::pair<Codec, AVScore>;
342 std::vector<CodecToScore> codecsToScores;
343
344 for (; it != storage.end() && it->id() == codecId; ++it) {
345 const AVScore score = scoreGetter ? scoreGetter(*it) : DefaultAVScore;
346 if (score != NotSuitableAVScore)
347 codecsToScores.emplace_back(args: *it, args: score);
348 }
349
350 if (scoreGetter) {
351 std::stable_sort(
352 codecsToScores.begin(), codecsToScores.end(),
353 [](const CodecToScore &a, const CodecToScore &b) { return a.second > b.second; });
354 }
355
356 auto open = [&opener](const CodecToScore &codecToScore) { return opener(codecToScore.first); };
357
358 return std::any_of(codecsToScores.begin(), codecsToScores.end(), open);
359}
360
361std::optional<Codec> findAVCodec(CodecStorageType codecsType, AVCodecID codecId,
362 const std::optional<PixelOrSampleFormat> &format)
363{
364 const CodecsStorage& storage = codecsStorage(codecsType);
365
366 // Storage is sorted, so we can quickly narrow down the search to codecs with the specific id.
367 auto begin = std::lower_bound(first: storage.begin(), last: storage.end(), val: codecId, comp: CodecsComparator{});
368 auto end = std::upper_bound(first: begin, last: storage.end(), val: codecId, comp: CodecsComparator{});
369
370 // Within the narrowed down range, look for a codec that supports the format.
371 // If no format is specified, return the first one.
372 auto codecIt = std::find_if(first: begin, last: end, pred: [&format](const Codec &codec) {
373 return !format || isAVFormatSupported(codec, format: *format);
374 });
375
376 if (codecIt != end)
377 return *codecIt;
378
379 return {};
380}
381
382} // namespace
383
384std::optional<Codec> findAVDecoder(AVCodecID codecId,
385 const std::optional<PixelOrSampleFormat> &format)
386{
387 return findAVCodec(codecsType: Decoders, codecId, format);
388}
389
390std::optional<Codec> findAVEncoder(AVCodecID codecId, const std::optional<PixelOrSampleFormat> &format)
391{
392 return findAVCodec(codecsType: Encoders, codecId, format);
393}
394
395bool findAndOpenAVDecoder(AVCodecID codecId,
396 const std::function<AVScore(const Codec &)> &scoresGetter,
397 const std::function<bool(const Codec &)> &codecOpener)
398{
399 return findAndOpenCodec(codecsType: Decoders, codecId, scoreGetter: scoresGetter, opener: codecOpener);
400}
401
402bool findAndOpenAVEncoder(AVCodecID codecId,
403 const std::function<AVScore(const Codec &)> &scoresGetter,
404 const std::function<bool(const Codec &)> &codecOpener)
405{
406 return findAndOpenCodec(codecsType: Encoders, codecId, scoreGetter: scoresGetter, opener: codecOpener);
407}
408
409} // namespace QFFmpeg
410
411QT_END_NAMESPACE
412

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpegcodecstorage.cpp