1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "libavutil/version.h"
5
6#include "qffmpeghwaccel_p.h"
7
8#if QT_CONFIG(wmf)
9# include "qffmpeghwaccel_d3d11_p.h"
10# include <QtCore/private/qsystemlibrary_p.h>
11#endif
12
13#include "qffmpeg_p.h"
14#include "qffmpegcodecstorage_p.h"
15#include "qffmpegmediaintegration_p.h"
16#include "qffmpegvideobuffer_p.h"
17#include "qscopedvaluerollback.h"
18
19#ifdef Q_OS_LINUX
20# include "QtCore/qfile.h"
21# include <QLibrary>
22#endif
23
24#include <rhi/qrhi.h>
25#include <qloggingcategory.h>
26#include <unordered_set>
27
28/* Infrastructure for HW acceleration goes into this file. */
29
30QT_BEGIN_NAMESPACE
31
32using namespace Qt::StringLiterals;
33
34static Q_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");
35
36namespace QFFmpeg {
37
38static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = {
39#if defined(Q_OS_ANDROID)
40 AV_HWDEVICE_TYPE_MEDIACODEC,
41#elif defined(Q_OS_LINUX)
42 AV_HWDEVICE_TYPE_CUDA,
43 AV_HWDEVICE_TYPE_VAAPI,
44
45 // TODO: investigate VDPAU advantages.
46 // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU
47 // if it's included into the ffmpeg build and vdpau drivers are installed.
48 // AV_HWDEVICE_TYPE_VDPAU
49#elif defined (Q_OS_WIN)
50 AV_HWDEVICE_TYPE_D3D11VA,
51#elif defined (Q_OS_DARWIN)
52 AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
53#endif
54};
55
56static AVBufferUPtr loadHWContext(AVHWDeviceType type)
57{
58 AVBufferRef *hwContext = nullptr;
59 qCDebug(qLHWAccel) << " Checking HW context:" << av_hwdevice_get_type_name(type);
60 int ret = av_hwdevice_ctx_create(device_ctx: &hwContext, type, device: nullptr, opts: nullptr, flags: 0);
61
62 if (ret == 0) {
63 qCDebug(qLHWAccel) << " Using above hw context.";
64 return AVBufferUPtr(hwContext);
65 }
66 qCDebug(qLHWAccel) << " Could not create hw context:" << ret << strerror(errnum: -ret);
67 return nullptr;
68}
69
70// FFmpeg might crash on loading non-existing hw devices.
71// Let's roughly precheck drivers/libraries.
72static bool precheckDriver(AVHWDeviceType type)
73{
74 // precheckings might need some improvements
75#if defined(Q_OS_LINUX)
76 if (type == AV_HWDEVICE_TYPE_CUDA) {
77 if (!QFile::exists(fileName: QLatin1String("/proc/driver/nvidia/version")))
78 return false;
79
80 // QTBUG-122199
81 // CUDA backend requires libnvcuvid in libavcodec
82 QLibrary lib(u"libnvcuvid.so"_s);
83 if (!lib.load())
84 return false;
85 lib.unload();
86 return true;
87 }
88#elif defined(Q_OS_WINDOWS)
89 if (type == AV_HWDEVICE_TYPE_D3D11VA)
90 return QSystemLibrary(QLatin1String("d3d11.dll")).load();
91
92#if QT_FFMPEG_HAS_D3D12VA
93 if (type == AV_HWDEVICE_TYPE_D3D12VA)
94 return QSystemLibrary(QLatin1String("d3d12.dll")).load();
95#endif
96
97 if (type == AV_HWDEVICE_TYPE_DXVA2)
98 return QSystemLibrary(QLatin1String("d3d9.dll")).load();
99
100 // TODO: check nvenc/nvdec and revisit the checking
101 if (type == AV_HWDEVICE_TYPE_CUDA)
102 return QSystemLibrary(QLatin1String("nvml.dll")).load();
103#else
104 Q_UNUSED(type);
105#endif
106
107 return true;
108}
109
110static bool checkHwType(AVHWDeviceType type)
111{
112 const auto deviceName = av_hwdevice_get_type_name(type);
113 if (!deviceName) {
114 qWarning() << "Internal FFmpeg error, unknow hw type:" << type;
115 return false;
116 }
117
118 if (!precheckDriver(type)) {
119 qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed";
120 return false;
121 }
122
123 if (type == AV_HWDEVICE_TYPE_MEDIACODEC ||
124 type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
125 type == AV_HWDEVICE_TYPE_D3D11VA ||
126#if QT_FFMPEG_HAS_D3D12VA
127 type == AV_HWDEVICE_TYPE_D3D12VA ||
128#endif
129 type == AV_HWDEVICE_TYPE_DXVA2)
130 return true; // Don't waste time; it's expected to work fine of the precheck is OK
131
132
133 QScopedValueRollback rollback(FFmpegLogsEnabledInThread);
134 FFmpegLogsEnabledInThread = false;
135
136 return loadHWContext(type) != nullptr;
137}
138
139static const std::vector<AVHWDeviceType> &deviceTypes()
140{
141 static const auto types = []() {
142 qCDebug(qLHWAccel) << "Check device types";
143 QElapsedTimer timer;
144 timer.start();
145
146 // gather hw pix formats
147 std::unordered_set<AVPixelFormat> hwPixFormats;
148 for (const Codec codec : CodecEnumerator()) {
149 forEachAVPixelFormat(codec, function: [&](AVPixelFormat format) {
150 if (isHwPixelFormat(format))
151 hwPixFormats.insert(x: format);
152 });
153 }
154
155 // create a device types list
156 std::vector<AVHWDeviceType> result;
157 AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
158 while ((type = av_hwdevice_iterate_types(prev: type)) != AV_HWDEVICE_TYPE_NONE)
159 if (hwPixFormats.count(x: pixelFormatForHwDevice(deviceType: type)) && checkHwType(type))
160 result.push_back(x: type);
161 result.shrink_to_fit();
162
163 // reorder the list accordingly preferredHardwareAccelerators
164 auto it = result.begin();
165 for (const auto preffered : preferredHardwareAccelerators) {
166 auto found = std::find(first: it, last: result.end(), val: preffered);
167 if (found != result.end())
168 std::rotate(first: it++, middle: found, last: std::next(x: found));
169 }
170
171 using namespace std::chrono;
172 qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(d: timer.durationElapsed());
173
174 return result;
175 }();
176
177 return types;
178}
179
180static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
181{
182 const auto definedDeviceTypes = qgetenv(varName: envVarName);
183
184 if (definedDeviceTypes.isNull())
185 return deviceTypes();
186
187 std::vector<AVHWDeviceType> result;
188 const auto definedDeviceTypesString = QString::fromUtf8(ba: definedDeviceTypes).toLower();
189 for (const auto &deviceType : definedDeviceTypesString.split(sep: u',')) {
190 if (!deviceType.isEmpty()) {
191 const auto foundType = av_hwdevice_find_type_by_name(name: deviceType.toUtf8().data());
192 if (foundType == AV_HWDEVICE_TYPE_NONE)
193 qWarning() << "Unknown hw device type" << deviceType;
194 else
195 result.emplace_back(args: foundType);
196 }
197 }
198
199 result.shrink_to_fit();
200 return result;
201}
202
203std::pair<std::optional<Codec>, HWAccelUPtr> HWAccel::findDecoderWithHwAccel(AVCodecID id)
204{
205 for (auto type : decodingDeviceTypes()) {
206 const std::optional<Codec> codec = findAVDecoder(codecId: id, format: pixelFormatForHwDevice(deviceType: type));
207
208 if (!codec)
209 continue;
210
211 qCDebug(qLHWAccel) << "Found potential codec" << codec->name() << "for hw accel" << type
212 << "; Checking the hw device...";
213
214 HWAccelUPtr hwAccel = create(deviceType: type);
215
216 if (!hwAccel)
217 continue;
218
219 qCDebug(qLHWAccel) << "HW device is OK";
220
221 return { codec, std::move(hwAccel) };
222 }
223
224 qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id;
225
226 return { std::nullopt, nullptr };
227}
228
229static bool isNoConversionFormat(AVPixelFormat f)
230{
231 bool needsConversion = true;
232 QFFmpegVideoBuffer::toQtPixelFormat(avPixelFormat: f, needsConversion: &needsConversion);
233 return !needsConversion;
234};
235
236// Used for the AVCodecContext::get_format callback
237AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *fmt)
238{
239 QSpan<const AVPixelFormat> suggestedFormats = makeSpan(values: fmt);
240 // First check HW accelerated codecs, the HW device context must be set
241 if (codecContext->hw_device_ctx) {
242 auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data;
243 ValueAndScore<AVPixelFormat> formatAndScore;
244
245 // to be rewritten via findBestAVFormat
246 const Codec codec{ codecContext->codec };
247 for (const AVCodecHWConfig *config : codec.hwConfigs()) {
248 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
249 continue;
250
251 if (device_ctx->type != config->device_type)
252 continue;
253
254 const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0;
255 const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE;
256
257 auto scoresGettor = [&](AVPixelFormat format) {
258 // check in supported codec->pix_fmts (avcodec_get_supported_config with
259 // AV_CODEC_CONFIG_PIX_FORMAT since n7.1); no reason to use findAVPixelFormat as
260 // we're already in the hw_config loop
261 const auto pixelFormats = codec.pixelFormats();
262 if (shouldCheckCodecFormats && !hasValue(range: pixelFormats, value: format))
263 return NotSuitableAVScore;
264
265 if (!shouldCheckCodecFormats && config->pix_fmt != format)
266 return NotSuitableAVScore;
267
268 auto result = DefaultAVScore;
269
270 if (isDeprecated)
271 result -= 10000;
272 if (isHwPixelFormat(format))
273 result += 10;
274
275 return result;
276 };
277
278 const auto found = findBestAVValueWithScore(values: suggestedFormats, calculateScore: scoresGettor);
279
280 if (found.score > formatAndScore.score)
281 formatAndScore = found;
282 }
283
284 const auto format = formatAndScore.value;
285 if (format) {
286 TextureConverter::applyDecoderPreset(format: *format, codecContext&: *codecContext);
287 qCDebug(qLHWAccel) << "Selected format" << *format << "for hw" << device_ctx->type;
288 return *format;
289 }
290 }
291
292 // prefer video formats we can handle directly
293 const auto noConversionFormat = findIf(range: suggestedFormats, predicate: &isNoConversionFormat);
294 if (noConversionFormat) {
295 qCDebug(qLHWAccel) << "Selected format with no conversion" << *noConversionFormat;
296 return *noConversionFormat;
297 }
298
299 const AVPixelFormat format = !suggestedFormats.empty() ? suggestedFormats[0] : AV_PIX_FMT_NONE;
300 qCDebug(qLHWAccel) << "Selected format with conversion" << format;
301
302 // take the native format, this will involve one additional format conversion on the CPU side
303 return format;
304}
305
306HWAccel::~HWAccel() = default;
307
308HWAccelUPtr HWAccel::create(AVHWDeviceType deviceType)
309{
310 if (auto ctx = loadHWContext(type: deviceType))
311 return HWAccelUPtr(new HWAccel(std::move(ctx)));
312 else
313 return {};
314}
315
316AVPixelFormat HWAccel::format(AVFrame *frame)
317{
318 if (!frame->hw_frames_ctx)
319 return AVPixelFormat(frame->format);
320
321 auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data;
322 Q_ASSERT(hwFramesContext);
323 return AVPixelFormat(hwFramesContext->sw_format);
324}
325
326const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes()
327{
328 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_ENCODING_HW_DEVICE_TYPES");
329 return result;
330}
331
332const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes()
333{
334 static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_DECODING_HW_DEVICE_TYPES");
335 return result;
336}
337
338AVHWDeviceContext *HWAccel::hwDeviceContext() const
339{
340 return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr;
341}
342
343AVPixelFormat HWAccel::hwFormat() const
344{
345 return pixelFormatForHwDevice(deviceType: deviceType());
346}
347
348const AVHWFramesConstraints *HWAccel::constraints() const
349{
350 std::call_once(once&: m_constraintsOnceFlag, f: [this]() {
351 if (auto context = hwDeviceContextAsBuffer())
352 m_constraints.reset(p: av_hwdevice_get_hwframe_constraints(ref: context, hwconfig: nullptr));
353 });
354
355 return m_constraints.get();
356}
357
358bool HWAccel::matchesSizeContraints(QSize size) const
359{
360 const auto constraints = this->constraints();
361 if (!constraints)
362 return true;
363
364 return size.width() >= constraints->min_width
365 && size.height() >= constraints->min_height
366 && size.width() <= constraints->max_width
367 && size.height() <= constraints->max_height;
368}
369
370AVHWDeviceType HWAccel::deviceType() const
371{
372 return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE;
373}
374
375void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size)
376{
377 if (m_hwFramesContext) {
378 qWarning() << "Frames context has been already created!";
379 return;
380 }
381
382 if (!m_hwDeviceContext)
383 return;
384
385 m_hwFramesContext.reset(p: av_hwframe_ctx_alloc(device_ctx: m_hwDeviceContext.get()));
386 auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
387 c->format = hwFormat();
388 c->sw_format = swFormat;
389 c->width = size.width();
390 c->height = size.height();
391 qCDebug(qLHWAccel) << "init frames context";
392 int err = av_hwframe_ctx_init(ref: m_hwFramesContext.get());
393 if (err < 0)
394 qWarning() << "failed to init HW frame context" << err << err2str(errnum: err);
395 else
396 qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format;
397}
398
399AVHWFramesContext *HWAccel::hwFramesContext() const
400{
401 return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr;
402}
403
404static void deleteHwFrameContextData(AVHWFramesContext *context)
405{
406 delete reinterpret_cast<HwFrameContextData *>(context->user_opaque);
407}
408
409HwFrameContextData &HwFrameContextData::ensure(AVFrame &hwFrame)
410{
411 Q_ASSERT(hwFrame.hw_frames_ctx && hwFrame.hw_frames_ctx->data);
412
413 auto context = reinterpret_cast<AVHWFramesContext *>(hwFrame.hw_frames_ctx->data);
414 if (!context->user_opaque) {
415 context->user_opaque = new HwFrameContextData;
416 Q_ASSERT(!context->free);
417 context->free = deleteHwFrameContextData;
418 } else {
419 Q_ASSERT(context->free == deleteHwFrameContextData);
420 }
421
422 return *reinterpret_cast<HwFrameContextData *>(context->user_opaque);
423}
424
425AVFrameUPtr copyFromHwPool(AVFrameUPtr frame)
426{
427#if QT_CONFIG(wmf)
428 return copyFromHwPoolD3D11(std::move(frame));
429#else
430 return frame;
431#endif
432}
433
434} // namespace QFFmpeg
435
436QT_END_NAMESPACE
437

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp