1// Copyright (C) 2024 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpegcodecstorage_p.h"
5
6#include "qffmpeg_p.h"
7#include "qffmpeghwaccel_p.h"
8
9#include <qdebug.h>
10#include <qloggingcategory.h>
11
12#include <algorithm>
13#include <vector>
14#include <array>
15
16#include <unordered_set>
17
18extern "C" {
19#include <libavutil/pixdesc.h>
20#include <libavutil/samplefmt.h>
21}
22
23#ifdef Q_OS_ANDROID
24# include <QtCore/qjniobject.h>
25# include <QtCore/qjniarray.h>
26# include <QtCore/qjnitypes.h>
27#endif
28
29QT_BEGIN_NAMESPACE
30
31#ifdef Q_OS_ANDROID
32Q_DECLARE_JNI_CLASS(QtVideoDeviceManager,
33 "org/qtproject/qt/android/multimedia/QtVideoDeviceManager")
34#endif // Q_OS_ANDROID
35
36static Q_LOGGING_CATEGORY(qLcCodecStorage, "qt.multimedia.ffmpeg.codecstorage");
37
38namespace QFFmpeg {
39
40namespace {
41
42enum CodecStorageType {
43 Encoders,
44 Decoders,
45
46 // TODO: maybe split sw/hw codecs
47
48 CodecStorageTypeCount
49};
50
51using CodecsStorage = std::vector<const AVCodec *>;
52
53struct CodecsComparator
54{
55 bool operator()(const AVCodec *a, const AVCodec *b) const
56 {
57 return a->id < b->id
58 || (a->id == b->id && isAVCodecExperimental(codec: a) < isAVCodecExperimental(codec: b));
59 }
60
61 bool operator()(const AVCodec *codec, AVCodecID id) const { return codec->id < id; }
62};
63
64template <typename FlagNames>
65QString flagsToString(int flags, const FlagNames &flagNames)
66{
67 QString result;
68 int leftover = flags;
69 for (const auto &flagAndName : flagNames)
70 if ((flags & flagAndName.first) != 0) {
71 leftover &= ~flagAndName.first;
72 if (!result.isEmpty())
73 result += ", ";
74 result += flagAndName.second;
75 }
76
77 if (leftover) {
78 if (!result.isEmpty())
79 result += ", ";
80 result += QString::number(leftover, base: 16);
81 }
82 return result;
83}
84
85void dumpCodecInfo(const AVCodec *codec)
86{
87 using FlagNames = std::initializer_list<std::pair<int, const char *>>;
88 const auto mediaType = codec->type == AVMEDIA_TYPE_VIDEO ? "video"
89 : codec->type == AVMEDIA_TYPE_AUDIO ? "audio"
90 : codec->type == AVMEDIA_TYPE_SUBTITLE ? "subtitle"
91 : "other_type";
92
93 const auto type = av_codec_is_encoder(codec)
94 ? av_codec_is_decoder(codec) ? "encoder/decoder:" : "encoder:"
95 : "decoder:";
96
97 static const FlagNames capabilitiesNames = {
98 { AV_CODEC_CAP_DRAW_HORIZ_BAND, "DRAW_HORIZ_BAND" },
99 { AV_CODEC_CAP_DR1, "DRAW_HORIZ_DR1" },
100 { AV_CODEC_CAP_DELAY, "DELAY" },
101 { AV_CODEC_CAP_SMALL_LAST_FRAME, "SMALL_LAST_FRAME" },
102 { AV_CODEC_CAP_SUBFRAMES, "SUBFRAMES" },
103 { AV_CODEC_CAP_EXPERIMENTAL, "EXPERIMENTAL" },
104 { AV_CODEC_CAP_CHANNEL_CONF, "CHANNEL_CONF" },
105 { AV_CODEC_CAP_FRAME_THREADS, "FRAME_THREADS" },
106 { AV_CODEC_CAP_SLICE_THREADS, "SLICE_THREADS" },
107 { AV_CODEC_CAP_PARAM_CHANGE, "PARAM_CHANGE" },
108#ifdef AV_CODEC_CAP_OTHER_THREADS
109 { AV_CODEC_CAP_OTHER_THREADS, "OTHER_THREADS" },
110#endif
111 { AV_CODEC_CAP_VARIABLE_FRAME_SIZE, "VARIABLE_FRAME_SIZE" },
112 { AV_CODEC_CAP_AVOID_PROBING, "AVOID_PROBING" },
113 { AV_CODEC_CAP_HARDWARE, "HARDWARE" },
114 { AV_CODEC_CAP_HYBRID, "HYBRID" },
115 { AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE, "ENCODER_REORDERED_OPAQUE" },
116#ifdef AV_CODEC_CAP_ENCODER_FLUSH
117 { AV_CODEC_CAP_ENCODER_FLUSH, "ENCODER_FLUSH" },
118#endif
119 };
120
121 qCDebug(qLcCodecStorage) << mediaType << type << codec->name << "id:" << codec->id
122 << "capabilities:"
123 << flagsToString(flags: codec->capabilities, flagNames: capabilitiesNames);
124
125 const auto pixelFormats = getCodecPixelFormats(codec);
126 if (pixelFormats) {
127 static const FlagNames flagNames = {
128 { AV_PIX_FMT_FLAG_BE, "BE" },
129 { AV_PIX_FMT_FLAG_PAL, "PAL" },
130 { AV_PIX_FMT_FLAG_BITSTREAM, "BITSTREAM" },
131 { AV_PIX_FMT_FLAG_HWACCEL, "HWACCEL" },
132 { AV_PIX_FMT_FLAG_PLANAR, "PLANAR" },
133 { AV_PIX_FMT_FLAG_RGB, "RGB" },
134 { AV_PIX_FMT_FLAG_ALPHA, "ALPHA" },
135 { AV_PIX_FMT_FLAG_BAYER, "BAYER" },
136 { AV_PIX_FMT_FLAG_FLOAT, "FLOAT" },
137 };
138
139 qCDebug(qLcCodecStorage) << " pixelFormats:";
140 for (auto f = pixelFormats; *f != AV_PIX_FMT_NONE; ++f) {
141 auto desc = av_pix_fmt_desc_get(pix_fmt: *f);
142 qCDebug(qLcCodecStorage)
143 << " id:" << *f << desc->name << "depth:" << desc->comp[0].depth
144 << "flags:" << flagsToString(flags: desc->flags, flagNames);
145 }
146 } else if (codec->type == AVMEDIA_TYPE_VIDEO) {
147 qCDebug(qLcCodecStorage) << " pixelFormats: null";
148 }
149
150 const auto sampleFormats = getCodecSampleFormats(codec);
151 if (sampleFormats) {
152 qCDebug(qLcCodecStorage) << " sampleFormats:";
153 for (auto f = sampleFormats; *f != AV_SAMPLE_FMT_NONE; ++f) {
154 const auto name = av_get_sample_fmt_name(sample_fmt: *f);
155 qCDebug(qLcCodecStorage) << " id:" << *f << (name ? name : "unknown")
156 << "bytes_per_sample:" << av_get_bytes_per_sample(sample_fmt: *f)
157 << "is_planar:" << av_sample_fmt_is_planar(sample_fmt: *f);
158 }
159 } else if (codec->type == AVMEDIA_TYPE_AUDIO) {
160 qCDebug(qLcCodecStorage) << " sampleFormats: null";
161 }
162
163 if (avcodec_get_hw_config(codec, index: 0)) {
164 static const FlagNames hwConfigMethodNames = {
165 { AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, "HW_DEVICE_CTX" },
166 { AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX, "HW_FRAMES_CTX" },
167 { AV_CODEC_HW_CONFIG_METHOD_INTERNAL, "INTERNAL" },
168 { AV_CODEC_HW_CONFIG_METHOD_AD_HOC, "AD_HOC" }
169 };
170
171 qCDebug(qLcCodecStorage) << " hw config:";
172 for (int index = 0; auto config = avcodec_get_hw_config(codec, index); ++index) {
173 const auto pixFmtForDevice = pixelFormatForHwDevice(deviceType: config->device_type);
174 auto pixFmtDesc = av_pix_fmt_desc_get(pix_fmt: config->pix_fmt);
175 auto pixFmtForDeviceDesc = av_pix_fmt_desc_get(pix_fmt: pixFmtForDevice);
176 qCDebug(qLcCodecStorage)
177 << " device_type:" << config->device_type << "pix_fmt:" << config->pix_fmt
178 << (pixFmtDesc ? pixFmtDesc->name : "unknown")
179 << "pixelFormatForHwDevice:" << pixelFormatForHwDevice(deviceType: config->device_type)
180 << (pixFmtForDeviceDesc ? pixFmtForDeviceDesc->name : "unknown")
181 << "hw_config_methods:" << flagsToString(flags: config->methods, flagNames: hwConfigMethodNames);
182 }
183 }
184}
185
186bool isCodecValid(const AVCodec *codec, const std::vector<AVHWDeviceType> &availableHwDeviceTypes,
187 const std::optional<std::unordered_set<AVCodecID>> &codecAvailableOnDevice)
188{
189 if (codec->type != AVMEDIA_TYPE_VIDEO)
190 return true;
191
192 const auto pixelFormats = getCodecPixelFormats(codec);
193 if (!pixelFormats) {
194#if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
195 // Disable V4L2 M2M codecs for encoding for now,
196 // TODO: Investigate on how to get them working
197 if (std::strstr(haystack: codec->name, needle: "_v4l2m2m") && av_codec_is_encoder(codec))
198 return false;
199
200 // MediaCodec in Android is used for hardware-accelerated media processing. That is why
201 // before marking it as valid, we need to make sure if it is available on current device.
202 if (std::strstr(haystack: codec->name, needle: "_mediacodec") && (codec->capabilities & AV_CODEC_CAP_HARDWARE)
203 && codecAvailableOnDevice && codecAvailableOnDevice->count(x: codec->id) == 0)
204 return false;
205#endif
206
207 return true; // To be investigated. This happens for RAW_VIDEO, that is supposed to be OK,
208 // and with v4l2m2m codecs, that is suspicious.
209 }
210
211 if (findAVPixelFormat(codec, predicate: &isHwPixelFormat) == AV_PIX_FMT_NONE)
212 return true;
213
214 if ((codec->capabilities & AV_CODEC_CAP_HARDWARE) == 0)
215 return true;
216
217 auto checkDeviceType = [codec](AVHWDeviceType type) {
218 return isAVFormatSupported(codec, format: pixelFormatForHwDevice(deviceType: type));
219 };
220
221 if (codecAvailableOnDevice && codecAvailableOnDevice->count(x: codec->id) == 0)
222 return false;
223
224 return std::any_of(first: availableHwDeviceTypes.begin(), last: availableHwDeviceTypes.end(),
225 pred: checkDeviceType);
226}
227
228std::optional<std::unordered_set<AVCodecID>> availableHWCodecs(const CodecStorageType type)
229{
230#ifdef Q_OS_ANDROID
231 using namespace Qt::StringLiterals;
232 using namespace QtJniTypes;
233 std::unordered_set<AVCodecID> availabeCodecs;
234
235 auto getCodecId = [](const QString &codecName) {
236 if (codecName == "3gpp"_L1)
237 return AV_CODEC_ID_H263;
238 if (codecName == "avc"_L1)
239 return AV_CODEC_ID_H264;
240 if (codecName == "hevc"_L1)
241 return AV_CODEC_ID_HEVC;
242 if (codecName == "mp4v-es"_L1)
243 return AV_CODEC_ID_MPEG4;
244 if (codecName == "x-vnd.on2.vp8"_L1)
245 return AV_CODEC_ID_VP8;
246 if (codecName == "x-vnd.on2.vp9"_L1)
247 return AV_CODEC_ID_VP9;
248 return AV_CODEC_ID_NONE;
249 };
250
251 const QJniArray jniCodecs = QtVideoDeviceManager::callStaticMethod<String[]>(
252 type == Encoders ? "getHWVideoEncoders" : "getHWVideoDecoders");
253
254 for (const auto &codec : jniCodecs)
255 availabeCodecs.insert(getCodecId(codec.toString()));
256 return availabeCodecs;
257#else
258 Q_UNUSED(type);
259 return {};
260#endif
261}
262
263const CodecsStorage &codecsStorage(CodecStorageType codecsType)
264{
265 static const auto &storages = []() {
266 std::array<CodecsStorage, CodecStorageTypeCount> result;
267 void *opaque = nullptr;
268 const auto platformHwEncoders = availableHWCodecs(type: Encoders);
269 const auto platformHwDecoders = availableHWCodecs(type: Decoders);
270
271 while (auto codec = av_codec_iterate(opaque: &opaque)) {
272 // TODO: to be investigated
273 // FFmpeg functions avcodec_find_decoder/avcodec_find_encoder
274 // find experimental codecs in the last order,
275 // now we don't consider them at all since they are supposed to
276 // be not stable, maybe we shouldn't.
277 // Currently, it's possible to turn them on for testing purposes.
278
279 static const auto experimentalCodecsEnabled =
280 qEnvironmentVariableIntValue(varName: "QT_ENABLE_EXPERIMENTAL_CODECS");
281
282 if (!experimentalCodecsEnabled && isAVCodecExperimental(codec)) {
283 qCDebug(qLcCodecStorage) << "Skip experimental codec" << codec->name;
284 continue;
285 }
286
287 if (av_codec_is_decoder(codec)) {
288 if (isCodecValid(codec, availableHwDeviceTypes: HWAccel::decodingDeviceTypes(), codecAvailableOnDevice: platformHwDecoders))
289 result[Decoders].emplace_back(args&: codec);
290 else
291 qCDebug(qLcCodecStorage)
292 << "Skip decoder" << codec->name
293 << "due to disabled matching hw acceleration, or dysfunctional codec";
294 }
295
296 if (av_codec_is_encoder(codec)) {
297 if (isCodecValid(codec, availableHwDeviceTypes: HWAccel::encodingDeviceTypes(), codecAvailableOnDevice: platformHwEncoders))
298 result[Encoders].emplace_back(args&: codec);
299 else
300 qCDebug(qLcCodecStorage)
301 << "Skip encoder" << codec->name
302 << "due to disabled matching hw acceleration, or dysfunctional codec";
303 }
304 }
305
306 for (auto &storage : result) {
307 storage.shrink_to_fit();
308
309 // we should ensure the original order
310 std::stable_sort(first: storage.begin(), last: storage.end(), comp: CodecsComparator{});
311 }
312
313 // It print pretty much logs, so let's print it only for special case
314 const bool shouldDumpCodecsInfo = qLcCodecStorage().isEnabled(type: QtDebugMsg)
315 && qEnvironmentVariableIsSet(varName: "QT_FFMPEG_DEBUG");
316
317 if (shouldDumpCodecsInfo) {
318 qCDebug(qLcCodecStorage) << "Advanced FFmpeg codecs info:";
319 for (auto &storage : result) {
320 std::for_each(first: storage.begin(), last: storage.end(), f: &dumpCodecInfo);
321 qCDebug(qLcCodecStorage) << "---------------------------";
322 }
323 }
324
325 return result;
326 }();
327
328 return storages[codecsType];
329}
330
331template <typename CodecScoreGetter, typename CodecOpener>
332bool findAndOpenCodec(CodecStorageType codecsType, AVCodecID codecId,
333 const CodecScoreGetter &scoreGetter, const CodecOpener &opener)
334{
335 Q_ASSERT(opener);
336 const auto &storage = codecsStorage(codecsType);
337 auto it = std::lower_bound(first: storage.begin(), last: storage.end(), val: codecId, comp: CodecsComparator{});
338
339 using CodecToScore = std::pair<const AVCodec *, AVScore>;
340 std::vector<CodecToScore> codecsToScores;
341
342 for (; it != storage.end() && (*it)->id == codecId; ++it) {
343 const AVScore score = scoreGetter ? scoreGetter(*it) : DefaultAVScore;
344 if (score != NotSuitableAVScore)
345 codecsToScores.emplace_back(args: *it, args: score);
346 }
347
348 if (scoreGetter) {
349 std::stable_sort(
350 codecsToScores.begin(), codecsToScores.end(),
351 [](const CodecToScore &a, const CodecToScore &b) { return a.second > b.second; });
352 }
353
354 auto open = [&opener](const CodecToScore &codecToScore) { return opener(codecToScore.first); };
355
356 return std::any_of(codecsToScores.begin(), codecsToScores.end(), open);
357}
358
359template <typename CodecScoreGetter>
360const AVCodec *findAVCodec(CodecStorageType codecsType, AVCodecID codecId,
361 const CodecScoreGetter &scoreGetter)
362{
363 const auto &storage = codecsStorage(codecsType);
364 auto it = std::lower_bound(first: storage.begin(), last: storage.end(), val: codecId, comp: CodecsComparator{});
365
366 const AVCodec *result = nullptr;
367 AVScore resultScore = NotSuitableAVScore;
368
369 for (; it != storage.end() && (*it)->id == codecId && resultScore != BestAVScore; ++it) {
370 const auto score = scoreGetter(*it);
371
372 if (score > resultScore) {
373 resultScore = score;
374 result = *it;
375 }
376 }
377
378 return result;
379}
380
381const AVCodec *findAVCodec(CodecStorageType codecsType, AVCodecID codecId,
382 const std::optional<PixelOrSampleFormat> &format)
383{
384 // TODO: remove deviceType and use only isAVFormatSupported to check the format
385
386 return findAVCodec(codecsType, codecId, scoreGetter: [&](const AVCodec *codec) {
387 if (format && !isAVFormatSupported(codec, format: *format))
388 return NotSuitableAVScore;
389
390 return BestAVScore;
391 });
392}
393
394} // namespace
395
396const AVCodec *findAVDecoder(AVCodecID codecId, const std::optional<PixelOrSampleFormat> &format)
397{
398 return findAVCodec(codecsType: Decoders, codecId, format);
399}
400
401const AVCodec *findAVEncoder(AVCodecID codecId, const std::optional<PixelOrSampleFormat> &format)
402{
403 return findAVCodec(codecsType: Encoders, codecId, format);
404}
405
406const AVCodec *findAVEncoder(AVCodecID codecId,
407 const std::function<AVScore(const AVCodec *)> &scoresGetter)
408{
409 return findAVCodec(codecsType: Encoders, codecId, scoreGetter: scoresGetter);
410}
411
412bool findAndOpenAVDecoder(AVCodecID codecId,
413 const std::function<AVScore(const AVCodec *)> &scoresGetter,
414 const std::function<bool(const AVCodec *)> &codecOpener)
415{
416 return findAndOpenCodec(codecsType: Decoders, codecId, scoreGetter: scoresGetter, opener: codecOpener);
417}
418
419bool findAndOpenAVEncoder(AVCodecID codecId,
420 const std::function<AVScore(const AVCodec *)> &scoresGetter,
421 const std::function<bool(const AVCodec *)> &codecOpener)
422{
423 return findAndOpenCodec(codecsType: Encoders, codecId, scoreGetter: scoresGetter, opener: codecOpener);
424}
425
426} // namespace QFFmpeg
427
428QT_END_NAMESPACE
429

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpegcodecstorage.cpp