1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "libavutil/version.h"
5
6#include "qffmpeghwaccel_p.h"
7#if QT_CONFIG(vaapi)
8#include "qffmpeghwaccel_vaapi_p.h"
9#endif
10#ifdef Q_OS_DARWIN
11#include "qffmpeghwaccel_videotoolbox_p.h"
12#endif
13#if QT_CONFIG(wmf)
14#include "qffmpeghwaccel_d3d11_p.h"
15#include <QtCore/private/qsystemlibrary_p.h>
16
17#endif
18#ifdef Q_OS_ANDROID
19# include "qffmpeghwaccel_mediacodec_p.h"
20#endif
21#include "qffmpeg_p.h"
22#include "qffmpegcodecstorage_p.h"
23#include "qffmpegvideobuffer_p.h"
24#include "qscopedvaluerollback.h"
25#include "QtCore/qfile.h"
26
27#include <rhi/qrhi.h>
28#include <qloggingcategory.h>
29#include <unordered_set>
30#ifdef Q_OS_LINUX
31#include <QLibrary>
32#endif
33
34/* Infrastructure for HW acceleration goes into this file. */
35
36QT_BEGIN_NAMESPACE
37
38static Q_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");
39extern bool thread_local FFmpegLogsEnabledInThread;
40
41namespace QFFmpeg {
42
43static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = {
44#if defined(Q_OS_ANDROID)
45 AV_HWDEVICE_TYPE_MEDIACODEC,
46#elif defined(Q_OS_LINUX)
47 AV_HWDEVICE_TYPE_CUDA,
48 AV_HWDEVICE_TYPE_VAAPI,
49
50 // TODO: investigate VDPAU advantages.
51 // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU
52 // if it's included into the ffmpeg build and vdpau drivers are installed.
53 // AV_HWDEVICE_TYPE_VDPAU
54#elif defined (Q_OS_WIN)
55 AV_HWDEVICE_TYPE_D3D11VA,
56#elif defined (Q_OS_DARWIN)
57 AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
58#endif
59};
60
61static AVBufferUPtr loadHWContext(AVHWDeviceType type)
62{
63 AVBufferRef *hwContext = nullptr;
64 qCDebug(qLHWAccel) << " Checking HW context:" << av_hwdevice_get_type_name(type);
65 int ret = av_hwdevice_ctx_create(device_ctx: &hwContext, type, device: nullptr, opts: nullptr, flags: 0);
66
67 if (ret == 0) {
68 qCDebug(qLHWAccel) << " Using above hw context.";
69 return AVBufferUPtr(hwContext);
70 }
71 qCDebug(qLHWAccel) << " Could not create hw context:" << ret << strerror(errnum: -ret);
72 return nullptr;
73}
74
75// FFmpeg might crash on loading non-existing hw devices.
76// Let's roughly precheck drivers/libraries.
77static bool precheckDriver(AVHWDeviceType type)
78{
79 // precheckings might need some improvements
80#if defined(Q_OS_LINUX)
81 if (type == AV_HWDEVICE_TYPE_CUDA) {
82 if (!QFile::exists(fileName: QLatin1String("/proc/driver/nvidia/version")))
83 return false;
84
85 // QTBUG-122199
86 // CUDA backend requires libnvcuvid in libavcodec
87 QLibrary lib("libnvcuvid.so");
88 if (!lib.load())
89 return false;
90 lib.unload();
91 return true;
92 }
93#elif defined(Q_OS_WINDOWS)
94 if (type == AV_HWDEVICE_TYPE_D3D11VA)
95 return QSystemLibrary(QLatin1String("d3d11.dll")).load();
96
97#if QT_FFMPEG_HAS_D3D12VA
98 if (type == AV_HWDEVICE_TYPE_D3D12VA)
99 return QSystemLibrary(QLatin1String("d3d12.dll")).load();
100#endif
101
102 if (type == AV_HWDEVICE_TYPE_DXVA2)
103 return QSystemLibrary(QLatin1String("d3d9.dll")).load();
104
105 // TODO: check nvenc/nvdec and revisit the checking
106 if (type == AV_HWDEVICE_TYPE_CUDA)
107 return QSystemLibrary(QLatin1String("nvml.dll")).load();
108#else
109 Q_UNUSED(type);
110#endif
111
112 return true;
113}
114
115static bool checkHwType(AVHWDeviceType type)
116{
117 const auto deviceName = av_hwdevice_get_type_name(type);
118 if (!deviceName) {
119 qWarning() << "Internal FFmpeg error, unknow hw type:" << type;
120 return false;
121 }
122
123 if (!precheckDriver(type)) {
124 qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed";
125 return false;
126 }
127
128 if (type == AV_HWDEVICE_TYPE_MEDIACODEC ||
129 type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
130 type == AV_HWDEVICE_TYPE_D3D11VA ||
131#if QT_FFMPEG_HAS_D3D12VA
132 type == AV_HWDEVICE_TYPE_D3D12VA ||
133#endif
134 type == AV_HWDEVICE_TYPE_DXVA2)
135 return true; // Don't waste time; it's expected to work fine of the precheck is OK
136
137
138 QScopedValueRollback rollback(FFmpegLogsEnabledInThread);
139 FFmpegLogsEnabledInThread = false;
140
141 return loadHWContext(type) != nullptr;
142}
143
144static const std::vector<AVHWDeviceType> &deviceTypes()
145{
146 static const auto types = []() {
147 qCDebug(qLHWAccel) << "Check device types";
148 QElapsedTimer timer;
149 timer.start();
150
151 // gather hw pix formats
152 std::unordered_set<AVPixelFormat> hwPixFormats;
153 void *opaque = nullptr;
154 while (auto codec = av_codec_iterate(opaque: &opaque)) {
155 findAVPixelFormat(codec, predicate: [&](AVPixelFormat format) {
156 if (isHwPixelFormat(format))
157 hwPixFormats.insert(x: format);
158 return false;
159 });
160 }
161
162 // create a device types list
163 std::vector<AVHWDeviceType> result;
164 AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
165 while ((type = av_hwdevice_iterate_types(prev: type)) != AV_HWDEVICE_TYPE_NONE)
166 if (hwPixFormats.count(x: pixelFormatForHwDevice(deviceType: type)) && checkHwType(type))
167 result.push_back(x: type);
168 result.shrink_to_fit();
169
170 // reorder the list accordingly preferredHardwareAccelerators
171 auto it = result.begin();
172 for (const auto preffered : preferredHardwareAccelerators) {
173 auto found = std::find(first: it, last: result.end(), val: preffered);
174 if (found != result.end())
175 std::rotate(first: it++, middle: found, last: std::next(x: found));
176 }
177
178 using namespace std::chrono;
179 qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(d: timer.durationElapsed());
180
181 return result;
182 }();
183
184 return types;
185}
186
187static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
188{
189 const auto definedDeviceTypes = qgetenv(varName: envVarName);
190
191 if (definedDeviceTypes.isNull())
192 return deviceTypes();
193
194 std::vector<AVHWDeviceType> result;
195 const auto definedDeviceTypesString = QString::fromUtf8(ba: definedDeviceTypes).toLower();
196 for (const auto &deviceType : definedDeviceTypesString.split(sep: ',')) {
197 if (!deviceType.isEmpty()) {
198 const auto foundType = av_hwdevice_find_type_by_name(name: deviceType.toUtf8().data());
199 if (foundType == AV_HWDEVICE_TYPE_NONE)
200 qWarning() << "Unknown hw device type" << deviceType;
201 else
202 result.emplace_back(args: foundType);
203 }
204 }
205
206 result.shrink_to_fit();
207 return result;
208}
209
210template <typename CodecFinder>
211std::pair<const AVCodec *, HWAccelUPtr>
212findCodecWithHwAccel(AVCodecID id, const std::vector<AVHWDeviceType> &deviceTypes,
213 CodecFinder codecFinder,
214 const std::function<bool(const HWAccel &)> &hwAccelPredicate)
215{
216 for (auto type : deviceTypes) {
217 const auto codec = codecFinder(id, pixelFormatForHwDevice(deviceType: type));
218
219 if (!codec)
220 continue;
221
222 qCDebug(qLHWAccel) << "Found potential codec" << codec->name << "for hw accel" << type
223 << "; Checking the hw device...";
224
225 auto hwAccel = QFFmpeg::HWAccel::create(deviceType: type);
226
227 if (!hwAccel)
228 continue;
229
230 if (hwAccelPredicate && !hwAccelPredicate(*hwAccel)) {
231 qCDebug(qLHWAccel) << "HW device is available but doesn't suit due to restrictions";
232 continue;
233 }
234
235 qCDebug(qLHWAccel) << "HW device is OK";
236
237 return { codec, std::move(hwAccel) };
238 }
239
240 qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id;
241
242 return { nullptr, nullptr };
243}
244
245static bool isNoConversionFormat(AVPixelFormat f)
246{
247 bool needsConversion = true;
248 QFFmpegVideoBuffer::toQtPixelFormat(avPixelFormat: f, needsConversion: &needsConversion);
249 return !needsConversion;
250};
251
252namespace {
253
254bool hwTextureConversionEnabled()
255{
256
257 // HW textures conversions are not stable in specific cases, dependent on the hardware and OS.
258 // We need the env var for testing with no textures conversion on the user's side.
259 static const int disableHwConversion =
260 qEnvironmentVariableIntValue(varName: "QT_DISABLE_HW_TEXTURES_CONVERSION");
261
262 return !disableHwConversion;
263}
264
265void setupDecoder(const AVPixelFormat format, AVCodecContext *const codecContext)
266{
267 if (!hwTextureConversionEnabled())
268 return;
269
270#if QT_CONFIG(wmf)
271 if (format == AV_PIX_FMT_D3D11)
272 QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(codecContext);
273#elif defined Q_OS_ANDROID
274 if (format == AV_PIX_FMT_MEDIACODEC)
275 QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(codecContext);
276#else
277 Q_UNUSED(codecContext);
278 Q_UNUSED(format);
279#endif
280}
281
282} // namespace
283
284// Used for the AVCodecContext::get_format callback
285AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *suggestedFormats)
286{
287 // First check HW accelerated codecs, the HW device context must be set
288 if (codecContext->hw_device_ctx) {
289 auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data;
290 std::pair formatAndScore(AV_PIX_FMT_NONE, NotSuitableAVScore);
291
292 // to be rewritten via findBestAVFormat
293 for (int i = 0;
294 const AVCodecHWConfig *config = avcodec_get_hw_config(codec: codecContext->codec, index: i); i++) {
295 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
296 continue;
297
298 if (device_ctx->type != config->device_type)
299 continue;
300
301 const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0;
302 const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE;
303
304 auto scoresGettor = [&](AVPixelFormat format) {
305 // check in supported codec->pix_fmts (avcodec_get_supported_config with
306 // AV_CODEC_CONFIG_PIX_FORMAT since n7.1); no reason to use findAVPixelFormat as
307 // we're already in the hw_config loop
308 const auto pixelFormats = getCodecPixelFormats(codec: codecContext->codec);
309 if (shouldCheckCodecFormats && !hasAVValue(fmts: pixelFormats, format))
310 return NotSuitableAVScore;
311
312 if (!shouldCheckCodecFormats && config->pix_fmt != format)
313 return NotSuitableAVScore;
314
315 auto result = DefaultAVScore;
316
317 if (isDeprecated)
318 result -= 10000;
319 if (isHwPixelFormat(format))
320 result += 10;
321
322 return result;
323 };
324
325 const auto found = findBestAVValue(values: suggestedFormats, calculateScore: scoresGettor);
326
327 if (found.second > formatAndScore.second)
328 formatAndScore = found;
329 }
330
331 const auto &format = formatAndScore.first;
332 if (format != AV_PIX_FMT_NONE) {
333 setupDecoder(format, codecContext);
334 qCDebug(qLHWAccel) << "Selected format" << format << "for hw" << device_ctx->type;
335 return format;
336 }
337 }
338
339 // prefer video formats we can handle directly
340 const auto noConversionFormat = findAVValue(fmts: suggestedFormats, predicate: &isNoConversionFormat);
341 if (noConversionFormat != AV_PIX_FMT_NONE) {
342 qCDebug(qLHWAccel) << "Selected format with no conversion" << noConversionFormat;
343 return noConversionFormat;
344 }
345
346 qCDebug(qLHWAccel) << "Selected format with conversion" << *suggestedFormats;
347
348 // take the native format, this will involve one additional format conversion on the CPU side
349 return *suggestedFormats;
350}
351
352HWAccel::~HWAccel() = default;
353
354HWAccelUPtr HWAccel::create(AVHWDeviceType deviceType)
355{
356 if (auto ctx = loadHWContext(type: deviceType))
357 return HWAccelUPtr(new HWAccel(std::move(ctx)));
358 else
359 return {};
360}
361
362AVPixelFormat HWAccel::format(AVFrame *frame)
363{
364 if (!frame->hw_frames_ctx)
365 return AVPixelFormat(frame->format);
366
367 auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data;
368 Q_ASSERT(hwFramesContext);
369 return AVPixelFormat(hwFramesContext->sw_format);
370}
371
372const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes()
373{
374 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_ENCODING_HW_DEVICE_TYPES");
375 return result;
376}
377
378const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes()
379{
380 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_DECODING_HW_DEVICE_TYPES");
381 return result;
382}
383
384AVHWDeviceContext *HWAccel::hwDeviceContext() const
385{
386 return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr;
387}
388
389AVPixelFormat HWAccel::hwFormat() const
390{
391 return pixelFormatForHwDevice(deviceType: deviceType());
392}
393
394const AVHWFramesConstraints *HWAccel::constraints() const
395{
396 std::call_once(once&: m_constraintsOnceFlag, f: [this]() {
397 if (auto context = hwDeviceContextAsBuffer())
398 m_constraints.reset(p: av_hwdevice_get_hwframe_constraints(ref: context, hwconfig: nullptr));
399 });
400
401 return m_constraints.get();
402}
403
404bool HWAccel::matchesSizeContraints(QSize size) const
405{
406 const auto constraints = this->constraints();
407 if (!constraints)
408 return true;
409
410 return size.width() >= constraints->min_width
411 && size.height() >= constraints->min_height
412 && size.width() <= constraints->max_width
413 && size.height() <= constraints->max_height;
414}
415
416std::pair<const AVCodec *, HWAccelUPtr>
417HWAccel::findEncoderWithHwAccel(AVCodecID id,
418 const std::function<bool(const HWAccel &)> &hwAccelPredicate)
419{
420 auto finder = qOverload<AVCodecID, const std::optional<PixelOrSampleFormat> &>(
421 &QFFmpeg::findAVEncoder);
422 return findCodecWithHwAccel(id, deviceTypes: encodingDeviceTypes(), codecFinder: finder, hwAccelPredicate);
423}
424
425std::pair<const AVCodec *, HWAccelUPtr>
426HWAccel::findDecoderWithHwAccel(AVCodecID id,
427 const std::function<bool(const HWAccel &)> &hwAccelPredicate)
428{
429 return findCodecWithHwAccel(id, deviceTypes: decodingDeviceTypes(), codecFinder: &QFFmpeg::findAVDecoder,
430 hwAccelPredicate);
431}
432
433AVHWDeviceType HWAccel::deviceType() const
434{
435 return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE;
436}
437
438void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size)
439{
440 if (m_hwFramesContext) {
441 qWarning() << "Frames context has been already created!";
442 return;
443 }
444
445 if (!m_hwDeviceContext)
446 return;
447
448 m_hwFramesContext.reset(p: av_hwframe_ctx_alloc(device_ctx: m_hwDeviceContext.get()));
449 auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
450 c->format = hwFormat();
451 c->sw_format = swFormat;
452 c->width = size.width();
453 c->height = size.height();
454 qCDebug(qLHWAccel) << "init frames context";
455 int err = av_hwframe_ctx_init(ref: m_hwFramesContext.get());
456 if (err < 0)
457 qWarning() << "failed to init HW frame context" << err << err2str(errnum: err);
458 else
459 qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format;
460}
461
462AVHWFramesContext *HWAccel::hwFramesContext() const
463{
464 return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr;
465}
466
467
468TextureConverter::TextureConverter(QRhi *rhi)
469 : d(new Data)
470{
471 d->rhi = rhi;
472}
473
474TextureSet *TextureConverter::getTextures(AVFrame *frame)
475{
476 if (!frame || isNull())
477 return nullptr;
478
479 Q_ASSERT(frame->format == d->format);
480 return d->backend->getTextures(frame);
481}
482
483void TextureConverter::updateBackend(AVPixelFormat fmt)
484{
485 d->backend = nullptr;
486 if (!d->rhi)
487 return;
488
489 if (!hwTextureConversionEnabled())
490 return;
491
492 switch (fmt) {
493#if QT_CONFIG(vaapi)
494 case AV_PIX_FMT_VAAPI:
495 d->backend = std::make_unique<VAAPITextureConverter>(args&: d->rhi);
496 break;
497#endif
498#ifdef Q_OS_DARWIN
499 case AV_PIX_FMT_VIDEOTOOLBOX:
500 d->backend = std::make_unique<VideoToolBoxTextureConverter>(d->rhi);
501 break;
502#endif
503#if QT_CONFIG(wmf)
504 case AV_PIX_FMT_D3D11:
505 d->backend = std::make_unique<D3D11TextureConverter>(d->rhi);
506 break;
507#endif
508#ifdef Q_OS_ANDROID
509 case AV_PIX_FMT_MEDIACODEC:
510 d->backend = std::make_unique<MediaCodecTextureConverter>(d->rhi);
511 break;
512#endif
513 default:
514 break;
515 }
516 d->format = fmt;
517}
518
519AVFrameUPtr copyFromHwPool(AVFrameUPtr frame)
520{
521#if QT_CONFIG(wmf)
522 return copyFromHwPoolD3D11(std::move(frame));
523#else
524 return frame;
525#endif
526}
527
528} // namespace QFFmpeg
529
530QT_END_NAMESPACE
531

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp