1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "libavutil/version.h"
5
6#include "qffmpeghwaccel_p.h"
7
8#ifdef Q_OS_WINDOWS
9# include "qffmpeghwaccel_d3d11_p.h"
10# include <QtCore/private/qsystemlibrary_p.h>
11#endif
12
13#include "qffmpeg_p.h"
14#include "qffmpegcodecstorage_p.h"
15#include "qffmpegmediaintegration_p.h"
16#include "qffmpegvideobuffer_p.h"
17#include "qscopedvaluerollback.h"
18
19#include <QtCore/QElapsedTimer>
20
21#ifdef Q_OS_LINUX
22# include "QtCore/qfile.h"
23# include <QLibrary>
24#endif
25
26#include <rhi/qrhi.h>
27#include <qloggingcategory.h>
28#include <unordered_set>
29
30/* Infrastructure for HW acceleration goes into this file. */
31
32QT_BEGIN_NAMESPACE
33
34using namespace Qt::StringLiterals;
35
36Q_STATIC_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");
37
38namespace QFFmpeg {
39
40static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = {
41#if defined(Q_OS_ANDROID)
42 AV_HWDEVICE_TYPE_MEDIACODEC,
43#elif defined(Q_OS_LINUX)
44 AV_HWDEVICE_TYPE_CUDA,
45 AV_HWDEVICE_TYPE_VAAPI,
46
47 // TODO: investigate VDPAU advantages.
48 // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU
49 // if it's included into the ffmpeg build and vdpau drivers are installed.
50 // AV_HWDEVICE_TYPE_VDPAU
51#elif defined (Q_OS_WIN)
52 AV_HWDEVICE_TYPE_D3D11VA,
53#elif defined (Q_OS_DARWIN)
54 AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
55#endif
56};
57
58static AVBufferUPtr loadHWContext(AVHWDeviceType type)
59{
60 AVBufferRef *hwContext = nullptr;
61 qCDebug(qLHWAccel) << " Checking HW context:" << av_hwdevice_get_type_name(type);
62 int ret = av_hwdevice_ctx_create(device_ctx: &hwContext, type, device: nullptr, opts: nullptr, flags: 0);
63
64 if (ret == 0) {
65 qCDebug(qLHWAccel) << " Using above hw context.";
66 return AVBufferUPtr(hwContext);
67 }
68 qCDebug(qLHWAccel) << " Could not create hw context:" << ret << strerror(errnum: -ret);
69 return nullptr;
70}
71
72// FFmpeg might crash on loading non-existing hw devices.
73// Let's roughly precheck drivers/libraries.
74static bool precheckDriver(AVHWDeviceType type)
75{
76 // precheckings might need some improvements
77#if defined(Q_OS_LINUX)
78 if (type == AV_HWDEVICE_TYPE_CUDA) {
79 if (!QFile::exists(fileName: QLatin1String("/proc/driver/nvidia/version")))
80 return false;
81
82 // QTBUG-122199
83 // CUDA backend requires libnvcuvid in libavcodec
84 QLibrary lib(u"libnvcuvid.so"_s);
85 if (!lib.load())
86 return false;
87 lib.unload();
88 return true;
89 }
90#elif defined(Q_OS_WINDOWS)
91 if (type == AV_HWDEVICE_TYPE_D3D11VA)
92 return QSystemLibrary(QLatin1String("d3d11.dll")).load();
93
94#if QT_FFMPEG_HAS_D3D12VA
95 if (type == AV_HWDEVICE_TYPE_D3D12VA)
96 return QSystemLibrary(QLatin1String("d3d12.dll")).load();
97#endif
98
99 if (type == AV_HWDEVICE_TYPE_DXVA2)
100 return QSystemLibrary(QLatin1String("d3d9.dll")).load();
101
102 // TODO: check nvenc/nvdec and revisit the checking
103 if (type == AV_HWDEVICE_TYPE_CUDA)
104 return QSystemLibrary(QLatin1String("nvml.dll")).load();
105#else
106 Q_UNUSED(type);
107#endif
108
109 return true;
110}
111
112static bool checkHwType(AVHWDeviceType type)
113{
114 const auto deviceName = av_hwdevice_get_type_name(type);
115 if (!deviceName) {
116 qWarning() << "Internal FFmpeg error, unknow hw type:" << type;
117 return false;
118 }
119
120 if (!precheckDriver(type)) {
121 qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed";
122 return false;
123 }
124
125 if (type == AV_HWDEVICE_TYPE_MEDIACODEC ||
126 type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
127 type == AV_HWDEVICE_TYPE_D3D11VA ||
128#if QT_FFMPEG_HAS_D3D12VA
129 type == AV_HWDEVICE_TYPE_D3D12VA ||
130#endif
131 type == AV_HWDEVICE_TYPE_DXVA2)
132 return true; // Don't waste time; it's expected to work fine of the precheck is OK
133
134
135 QScopedValueRollback rollback(FFmpegLogsEnabledInThread);
136 FFmpegLogsEnabledInThread = false;
137
138 return loadHWContext(type) != nullptr;
139}
140
141static const std::vector<AVHWDeviceType> &deviceTypes()
142{
143 static const auto types = []() {
144 qCDebug(qLHWAccel) << "Check device types";
145 QElapsedTimer timer;
146 timer.start();
147
148 // gather hw pix formats
149 std::unordered_set<AVPixelFormat> hwPixFormats;
150 for (const Codec codec : CodecEnumerator()) {
151 forEachAVPixelFormat(codec, function: [&](AVPixelFormat format) {
152 if (isHwPixelFormat(format))
153 hwPixFormats.insert(x: format);
154 });
155 }
156
157 // create a device types list
158 std::vector<AVHWDeviceType> result;
159 AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
160 while ((type = av_hwdevice_iterate_types(prev: type)) != AV_HWDEVICE_TYPE_NONE)
161 if (hwPixFormats.count(x: pixelFormatForHwDevice(deviceType: type)) && checkHwType(type))
162 result.push_back(x: type);
163 result.shrink_to_fit();
164
165 // reorder the list accordingly preferredHardwareAccelerators
166 auto it = result.begin();
167 for (const auto preffered : preferredHardwareAccelerators) {
168 auto found = std::find(first: it, last: result.end(), val: preffered);
169 if (found != result.end())
170 std::rotate(first: it++, middle: found, last: std::next(x: found));
171 }
172
173 using namespace std::chrono;
174 qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(d: timer.durationElapsed());
175
176 return result;
177 }();
178
179 return types;
180}
181
182static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
183{
184 const auto definedDeviceTypes = qgetenv(varName: envVarName);
185
186 if (definedDeviceTypes.isNull())
187 return deviceTypes();
188
189 std::vector<AVHWDeviceType> result;
190 const auto definedDeviceTypesString = QString::fromUtf8(ba: definedDeviceTypes).toLower();
191 for (const auto &deviceType : definedDeviceTypesString.split(sep: u',')) {
192 if (!deviceType.isEmpty()) {
193 const auto foundType = av_hwdevice_find_type_by_name(name: deviceType.toUtf8().data());
194 if (foundType == AV_HWDEVICE_TYPE_NONE)
195 qWarning() << "Unknown hw device type" << deviceType;
196 else
197 result.emplace_back(args: foundType);
198 }
199 }
200
201 result.shrink_to_fit();
202 return result;
203}
204
205std::pair<std::optional<Codec>, HWAccelUPtr> HWAccel::findDecoderWithHwAccel(AVCodecID id)
206{
207 for (auto type : decodingDeviceTypes()) {
208 const std::optional<Codec> codec = findAVDecoder(codecId: id, format: pixelFormatForHwDevice(deviceType: type));
209
210 if (!codec)
211 continue;
212
213 qCDebug(qLHWAccel) << "Found potential codec" << codec->name() << "for hw accel" << type
214 << "; Checking the hw device...";
215
216 HWAccelUPtr hwAccel = create(deviceType: type);
217
218 if (!hwAccel)
219 continue;
220
221 qCDebug(qLHWAccel) << "HW device is OK";
222
223 return { codec, std::move(hwAccel) };
224 }
225
226 qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id;
227
228 return { std::nullopt, nullptr };
229}
230
231static bool isNoConversionFormat(AVPixelFormat f)
232{
233 bool needsConversion = true;
234 QFFmpegVideoBuffer::toQtPixelFormat(avPixelFormat: f, needsConversion: &needsConversion);
235 return !needsConversion;
236};
237
238// Used for the AVCodecContext::get_format callback
239AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *fmt)
240{
241 QSpan<const AVPixelFormat> suggestedFormats = makeSpan(values: fmt);
242 // First check HW accelerated codecs, the HW device context must be set
243 if (codecContext->hw_device_ctx) {
244 auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data;
245 ValueAndScore<AVPixelFormat> formatAndScore;
246
247 // to be rewritten via findBestAVFormat
248 const Codec codec{ codecContext->codec };
249 for (const AVCodecHWConfig *config : codec.hwConfigs()) {
250 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
251 continue;
252
253 if (device_ctx->type != config->device_type)
254 continue;
255
256 const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0;
257 const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE;
258
259 auto scoresGettor = [&](AVPixelFormat format) {
260 // check in supported codec->pix_fmts (avcodec_get_supported_config with
261 // AV_CODEC_CONFIG_PIX_FORMAT since n7.1); no reason to use findAVPixelFormat as
262 // we're already in the hw_config loop
263 const auto pixelFormats = codec.pixelFormats();
264 if (shouldCheckCodecFormats && !hasValue(range: pixelFormats, value: format))
265 return NotSuitableAVScore;
266
267 if (!shouldCheckCodecFormats && config->pix_fmt != format)
268 return NotSuitableAVScore;
269
270 auto result = DefaultAVScore;
271
272 if (isDeprecated)
273 result -= 10000;
274 if (isHwPixelFormat(format))
275 result += 10;
276
277 return result;
278 };
279
280 const auto found = findBestAVValueWithScore(values: suggestedFormats, calculateScore: scoresGettor);
281
282 if (found.score > formatAndScore.score)
283 formatAndScore = found;
284 }
285
286 const auto format = formatAndScore.value;
287 if (format) {
288 TextureConverter::applyDecoderPreset(format: *format, codecContext&: *codecContext);
289 qCDebug(qLHWAccel) << "Selected format" << *format << "for hw" << device_ctx->type;
290 return *format;
291 }
292 }
293
294 // prefer video formats we can handle directly
295 const auto noConversionFormat = findIf(range: suggestedFormats, predicate: &isNoConversionFormat);
296 if (noConversionFormat) {
297 qCDebug(qLHWAccel) << "Selected format with no conversion" << *noConversionFormat;
298 return *noConversionFormat;
299 }
300
301 const AVPixelFormat format = !suggestedFormats.empty() ? suggestedFormats[0] : AV_PIX_FMT_NONE;
302 qCDebug(qLHWAccel) << "Selected format with conversion" << format;
303
304 // take the native format, this will involve one additional format conversion on the CPU side
305 return format;
306}
307
308HWAccel::~HWAccel() = default;
309
310HWAccelUPtr HWAccel::create(AVHWDeviceType deviceType)
311{
312 if (auto ctx = loadHWContext(type: deviceType))
313 return HWAccelUPtr(new HWAccel(std::move(ctx)));
314 else
315 return {};
316}
317
318AVPixelFormat HWAccel::format(AVFrame *frame)
319{
320 if (!frame->hw_frames_ctx)
321 return AVPixelFormat(frame->format);
322
323 auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data;
324 Q_ASSERT(hwFramesContext);
325 return AVPixelFormat(hwFramesContext->sw_format);
326}
327
328const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes()
329{
330 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_ENCODING_HW_DEVICE_TYPES");
331 return result;
332}
333
334const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes()
335{
336 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_DECODING_HW_DEVICE_TYPES");
337 return result;
338}
339
340AVHWDeviceContext *HWAccel::hwDeviceContext() const
341{
342 return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr;
343}
344
345AVPixelFormat HWAccel::hwFormat() const
346{
347 return pixelFormatForHwDevice(deviceType: deviceType());
348}
349
350const AVHWFramesConstraints *HWAccel::constraints() const
351{
352 std::call_once(once&: m_constraintsOnceFlag, f: [this]() {
353 if (auto context = hwDeviceContextAsBuffer())
354 m_constraints.reset(p: av_hwdevice_get_hwframe_constraints(ref: context, hwconfig: nullptr));
355 });
356
357 return m_constraints.get();
358}
359
360bool HWAccel::matchesSizeContraints(QSize size) const
361{
362 const auto constraints = this->constraints();
363 if (!constraints)
364 return true;
365
366 return size.width() >= constraints->min_width
367 && size.height() >= constraints->min_height
368 && size.width() <= constraints->max_width
369 && size.height() <= constraints->max_height;
370}
371
372AVHWDeviceType HWAccel::deviceType() const
373{
374 return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE;
375}
376
377void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size)
378{
379 if (m_hwFramesContext) {
380 qWarning() << "Frames context has been already created!";
381 return;
382 }
383
384 if (!m_hwDeviceContext)
385 return;
386
387 m_hwFramesContext.reset(p: av_hwframe_ctx_alloc(device_ctx: m_hwDeviceContext.get()));
388 auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
389 c->format = hwFormat();
390 c->sw_format = swFormat;
391 c->width = size.width();
392 c->height = size.height();
393 qCDebug(qLHWAccel) << "init frames context";
394 int err = av_hwframe_ctx_init(ref: m_hwFramesContext.get());
395 if (err < 0)
396 qWarning() << "failed to init HW frame context" << err << AVError(err);
397 else
398 qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format;
399}
400
401AVHWFramesContext *HWAccel::hwFramesContext() const
402{
403 return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr;
404}
405
406static void deleteHwFrameContextData(AVHWFramesContext *context)
407{
408 delete reinterpret_cast<HwFrameContextData *>(context->user_opaque);
409}
410
411HwFrameContextData &HwFrameContextData::ensure(AVFrame &hwFrame)
412{
413 Q_ASSERT(hwFrame.hw_frames_ctx && hwFrame.hw_frames_ctx->data);
414
415 auto context = reinterpret_cast<AVHWFramesContext *>(hwFrame.hw_frames_ctx->data);
416 if (!context->user_opaque) {
417 context->user_opaque = new HwFrameContextData;
418 Q_ASSERT(!context->free);
419 context->free = deleteHwFrameContextData;
420 } else {
421 Q_ASSERT(context->free == deleteHwFrameContextData);
422 }
423
424 return *reinterpret_cast<HwFrameContextData *>(context->user_opaque);
425}
426
427AVFrameUPtr copyFromHwPool(AVFrameUPtr frame)
428{
429#ifdef Q_OS_WINDOWS
430 return copyFromHwPoolD3D11(std::move(frame));
431#else
432 return frame;
433#endif
434}
435
436} // namespace QFFmpeg
437
438QT_END_NAMESPACE
439

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp