1 | // Copyright (C) 2021 The Qt Company Ltd. |
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | |
4 | #include "libavutil/version.h" |
5 | |
6 | #include "qffmpeghwaccel_p.h" |
7 | |
8 | #if QT_CONFIG(wmf) |
9 | # include "qffmpeghwaccel_d3d11_p.h" |
10 | # include <QtCore/private/qsystemlibrary_p.h> |
11 | #endif |
12 | |
13 | #include "qffmpeg_p.h" |
14 | #include "qffmpegcodecstorage_p.h" |
15 | #include "qffmpegmediaintegration_p.h" |
16 | #include "qffmpegvideobuffer_p.h" |
17 | #include "qscopedvaluerollback.h" |
18 | |
19 | #ifdef Q_OS_LINUX |
20 | # include "QtCore/qfile.h" |
21 | # include <QLibrary> |
22 | #endif |
23 | |
24 | #include <rhi/qrhi.h> |
25 | #include <qloggingcategory.h> |
26 | #include <unordered_set> |
27 | |
28 | /* Infrastructure for HW acceleration goes into this file. */ |
29 | |
30 | QT_BEGIN_NAMESPACE |
31 | |
32 | using namespace Qt::StringLiterals; |
33 | |
34 | static Q_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel" ); |
35 | |
36 | namespace QFFmpeg { |
37 | |
38 | static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = { |
39 | #if defined(Q_OS_ANDROID) |
40 | AV_HWDEVICE_TYPE_MEDIACODEC, |
41 | #elif defined(Q_OS_LINUX) |
42 | AV_HWDEVICE_TYPE_CUDA, |
43 | AV_HWDEVICE_TYPE_VAAPI, |
44 | |
45 | // TODO: investigate VDPAU advantages. |
46 | // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU |
47 | // if it's included into the ffmpeg build and vdpau drivers are installed. |
48 | // AV_HWDEVICE_TYPE_VDPAU |
49 | #elif defined (Q_OS_WIN) |
50 | AV_HWDEVICE_TYPE_D3D11VA, |
51 | #elif defined (Q_OS_DARWIN) |
52 | AV_HWDEVICE_TYPE_VIDEOTOOLBOX, |
53 | #endif |
54 | }; |
55 | |
56 | static AVBufferUPtr loadHWContext(AVHWDeviceType type) |
57 | { |
58 | AVBufferRef *hwContext = nullptr; |
59 | qCDebug(qLHWAccel) << " Checking HW context:" << av_hwdevice_get_type_name(type); |
60 | int ret = av_hwdevice_ctx_create(device_ctx: &hwContext, type, device: nullptr, opts: nullptr, flags: 0); |
61 | |
62 | if (ret == 0) { |
63 | qCDebug(qLHWAccel) << " Using above hw context." ; |
64 | return AVBufferUPtr(hwContext); |
65 | } |
66 | qCDebug(qLHWAccel) << " Could not create hw context:" << ret << strerror(errnum: -ret); |
67 | return nullptr; |
68 | } |
69 | |
70 | // FFmpeg might crash on loading non-existing hw devices. |
71 | // Let's roughly precheck drivers/libraries. |
72 | static bool precheckDriver(AVHWDeviceType type) |
73 | { |
74 | // precheckings might need some improvements |
75 | #if defined(Q_OS_LINUX) |
76 | if (type == AV_HWDEVICE_TYPE_CUDA) { |
77 | if (!QFile::exists(fileName: QLatin1String("/proc/driver/nvidia/version" ))) |
78 | return false; |
79 | |
80 | // QTBUG-122199 |
81 | // CUDA backend requires libnvcuvid in libavcodec |
82 | QLibrary lib(u"libnvcuvid.so"_s ); |
83 | if (!lib.load()) |
84 | return false; |
85 | lib.unload(); |
86 | return true; |
87 | } |
88 | #elif defined(Q_OS_WINDOWS) |
89 | if (type == AV_HWDEVICE_TYPE_D3D11VA) |
90 | return QSystemLibrary(QLatin1String("d3d11.dll" )).load(); |
91 | |
92 | #if QT_FFMPEG_HAS_D3D12VA |
93 | if (type == AV_HWDEVICE_TYPE_D3D12VA) |
94 | return QSystemLibrary(QLatin1String("d3d12.dll" )).load(); |
95 | #endif |
96 | |
97 | if (type == AV_HWDEVICE_TYPE_DXVA2) |
98 | return QSystemLibrary(QLatin1String("d3d9.dll" )).load(); |
99 | |
100 | // TODO: check nvenc/nvdec and revisit the checking |
101 | if (type == AV_HWDEVICE_TYPE_CUDA) |
102 | return QSystemLibrary(QLatin1String("nvml.dll" )).load(); |
103 | #else |
104 | Q_UNUSED(type); |
105 | #endif |
106 | |
107 | return true; |
108 | } |
109 | |
110 | static bool checkHwType(AVHWDeviceType type) |
111 | { |
112 | const auto deviceName = av_hwdevice_get_type_name(type); |
113 | if (!deviceName) { |
114 | qWarning() << "Internal FFmpeg error, unknow hw type:" << type; |
115 | return false; |
116 | } |
117 | |
118 | if (!precheckDriver(type)) { |
119 | qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed" ; |
120 | return false; |
121 | } |
122 | |
123 | if (type == AV_HWDEVICE_TYPE_MEDIACODEC || |
124 | type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX || |
125 | type == AV_HWDEVICE_TYPE_D3D11VA || |
126 | #if QT_FFMPEG_HAS_D3D12VA |
127 | type == AV_HWDEVICE_TYPE_D3D12VA || |
128 | #endif |
129 | type == AV_HWDEVICE_TYPE_DXVA2) |
130 | return true; // Don't waste time; it's expected to work fine of the precheck is OK |
131 | |
132 | |
133 | QScopedValueRollback rollback(FFmpegLogsEnabledInThread); |
134 | FFmpegLogsEnabledInThread = false; |
135 | |
136 | return loadHWContext(type) != nullptr; |
137 | } |
138 | |
139 | static const std::vector<AVHWDeviceType> &deviceTypes() |
140 | { |
141 | static const auto types = []() { |
142 | qCDebug(qLHWAccel) << "Check device types" ; |
143 | QElapsedTimer timer; |
144 | timer.start(); |
145 | |
146 | // gather hw pix formats |
147 | std::unordered_set<AVPixelFormat> hwPixFormats; |
148 | for (const Codec codec : CodecEnumerator()) { |
149 | forEachAVPixelFormat(codec, function: [&](AVPixelFormat format) { |
150 | if (isHwPixelFormat(format)) |
151 | hwPixFormats.insert(x: format); |
152 | }); |
153 | } |
154 | |
155 | // create a device types list |
156 | std::vector<AVHWDeviceType> result; |
157 | AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE; |
158 | while ((type = av_hwdevice_iterate_types(prev: type)) != AV_HWDEVICE_TYPE_NONE) |
159 | if (hwPixFormats.count(x: pixelFormatForHwDevice(deviceType: type)) && checkHwType(type)) |
160 | result.push_back(x: type); |
161 | result.shrink_to_fit(); |
162 | |
163 | // reorder the list accordingly preferredHardwareAccelerators |
164 | auto it = result.begin(); |
165 | for (const auto preffered : preferredHardwareAccelerators) { |
166 | auto found = std::find(first: it, last: result.end(), val: preffered); |
167 | if (found != result.end()) |
168 | std::rotate(first: it++, middle: found, last: std::next(x: found)); |
169 | } |
170 | |
171 | using namespace std::chrono; |
172 | qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(d: timer.durationElapsed()); |
173 | |
174 | return result; |
175 | }(); |
176 | |
177 | return types; |
178 | } |
179 | |
180 | static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName) |
181 | { |
182 | const auto definedDeviceTypes = qgetenv(varName: envVarName); |
183 | |
184 | if (definedDeviceTypes.isNull()) |
185 | return deviceTypes(); |
186 | |
187 | std::vector<AVHWDeviceType> result; |
188 | const auto definedDeviceTypesString = QString::fromUtf8(ba: definedDeviceTypes).toLower(); |
189 | for (const auto &deviceType : definedDeviceTypesString.split(sep: u',')) { |
190 | if (!deviceType.isEmpty()) { |
191 | const auto foundType = av_hwdevice_find_type_by_name(name: deviceType.toUtf8().data()); |
192 | if (foundType == AV_HWDEVICE_TYPE_NONE) |
193 | qWarning() << "Unknown hw device type" << deviceType; |
194 | else |
195 | result.emplace_back(args: foundType); |
196 | } |
197 | } |
198 | |
199 | result.shrink_to_fit(); |
200 | return result; |
201 | } |
202 | |
203 | std::pair<std::optional<Codec>, HWAccelUPtr> HWAccel::findDecoderWithHwAccel(AVCodecID id) |
204 | { |
205 | for (auto type : decodingDeviceTypes()) { |
206 | const std::optional<Codec> codec = findAVDecoder(codecId: id, format: pixelFormatForHwDevice(deviceType: type)); |
207 | |
208 | if (!codec) |
209 | continue; |
210 | |
211 | qCDebug(qLHWAccel) << "Found potential codec" << codec->name() << "for hw accel" << type |
212 | << "; Checking the hw device..." ; |
213 | |
214 | HWAccelUPtr hwAccel = create(deviceType: type); |
215 | |
216 | if (!hwAccel) |
217 | continue; |
218 | |
219 | qCDebug(qLHWAccel) << "HW device is OK" ; |
220 | |
221 | return { codec, std::move(hwAccel) }; |
222 | } |
223 | |
224 | qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id; |
225 | |
226 | return { std::nullopt, nullptr }; |
227 | } |
228 | |
229 | static bool isNoConversionFormat(AVPixelFormat f) |
230 | { |
231 | bool needsConversion = true; |
232 | QFFmpegVideoBuffer::toQtPixelFormat(avPixelFormat: f, needsConversion: &needsConversion); |
233 | return !needsConversion; |
234 | }; |
235 | |
236 | // Used for the AVCodecContext::get_format callback |
237 | AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *fmt) |
238 | { |
239 | QSpan<const AVPixelFormat> suggestedFormats = makeSpan(values: fmt); |
240 | // First check HW accelerated codecs, the HW device context must be set |
241 | if (codecContext->hw_device_ctx) { |
242 | auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data; |
243 | ValueAndScore<AVPixelFormat> formatAndScore; |
244 | |
245 | // to be rewritten via findBestAVFormat |
246 | const Codec codec{ codecContext->codec }; |
247 | for (const AVCodecHWConfig *config : codec.hwConfigs()) { |
248 | if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) |
249 | continue; |
250 | |
251 | if (device_ctx->type != config->device_type) |
252 | continue; |
253 | |
254 | const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0; |
255 | const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE; |
256 | |
257 | auto scoresGettor = [&](AVPixelFormat format) { |
258 | // check in supported codec->pix_fmts (avcodec_get_supported_config with |
259 | // AV_CODEC_CONFIG_PIX_FORMAT since n7.1); no reason to use findAVPixelFormat as |
260 | // we're already in the hw_config loop |
261 | const auto pixelFormats = codec.pixelFormats(); |
262 | if (shouldCheckCodecFormats && !hasValue(range: pixelFormats, value: format)) |
263 | return NotSuitableAVScore; |
264 | |
265 | if (!shouldCheckCodecFormats && config->pix_fmt != format) |
266 | return NotSuitableAVScore; |
267 | |
268 | auto result = DefaultAVScore; |
269 | |
270 | if (isDeprecated) |
271 | result -= 10000; |
272 | if (isHwPixelFormat(format)) |
273 | result += 10; |
274 | |
275 | return result; |
276 | }; |
277 | |
278 | const auto found = findBestAVValueWithScore(values: suggestedFormats, calculateScore: scoresGettor); |
279 | |
280 | if (found.score > formatAndScore.score) |
281 | formatAndScore = found; |
282 | } |
283 | |
284 | const auto format = formatAndScore.value; |
285 | if (format) { |
286 | TextureConverter::applyDecoderPreset(format: *format, codecContext&: *codecContext); |
287 | qCDebug(qLHWAccel) << "Selected format" << *format << "for hw" << device_ctx->type; |
288 | return *format; |
289 | } |
290 | } |
291 | |
292 | // prefer video formats we can handle directly |
293 | const auto noConversionFormat = findIf(range: suggestedFormats, predicate: &isNoConversionFormat); |
294 | if (noConversionFormat) { |
295 | qCDebug(qLHWAccel) << "Selected format with no conversion" << *noConversionFormat; |
296 | return *noConversionFormat; |
297 | } |
298 | |
299 | const AVPixelFormat format = !suggestedFormats.empty() ? suggestedFormats[0] : AV_PIX_FMT_NONE; |
300 | qCDebug(qLHWAccel) << "Selected format with conversion" << format; |
301 | |
302 | // take the native format, this will involve one additional format conversion on the CPU side |
303 | return format; |
304 | } |
305 | |
306 | HWAccel::~HWAccel() = default; |
307 | |
308 | HWAccelUPtr HWAccel::create(AVHWDeviceType deviceType) |
309 | { |
310 | if (auto ctx = loadHWContext(type: deviceType)) |
311 | return HWAccelUPtr(new HWAccel(std::move(ctx))); |
312 | else |
313 | return {}; |
314 | } |
315 | |
316 | AVPixelFormat HWAccel::format(AVFrame *frame) |
317 | { |
318 | if (!frame->hw_frames_ctx) |
319 | return AVPixelFormat(frame->format); |
320 | |
321 | auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data; |
322 | Q_ASSERT(hwFramesContext); |
323 | return AVPixelFormat(hwFramesContext->sw_format); |
324 | } |
325 | |
326 | const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes() |
327 | { |
328 | static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_ENCODING_HW_DEVICE_TYPES" ); |
329 | return result; |
330 | } |
331 | |
332 | const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes() |
333 | { |
334 | static const auto &result = deviceTypes(envVarName: "QT_FFMPEG_DECODING_HW_DEVICE_TYPES" ); |
335 | return result; |
336 | } |
337 | |
338 | AVHWDeviceContext *HWAccel::hwDeviceContext() const |
339 | { |
340 | return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr; |
341 | } |
342 | |
343 | AVPixelFormat HWAccel::hwFormat() const |
344 | { |
345 | return pixelFormatForHwDevice(deviceType: deviceType()); |
346 | } |
347 | |
348 | const AVHWFramesConstraints *HWAccel::constraints() const |
349 | { |
350 | std::call_once(once&: m_constraintsOnceFlag, f: [this]() { |
351 | if (auto context = hwDeviceContextAsBuffer()) |
352 | m_constraints.reset(p: av_hwdevice_get_hwframe_constraints(ref: context, hwconfig: nullptr)); |
353 | }); |
354 | |
355 | return m_constraints.get(); |
356 | } |
357 | |
358 | bool HWAccel::matchesSizeContraints(QSize size) const |
359 | { |
360 | const auto constraints = this->constraints(); |
361 | if (!constraints) |
362 | return true; |
363 | |
364 | return size.width() >= constraints->min_width |
365 | && size.height() >= constraints->min_height |
366 | && size.width() <= constraints->max_width |
367 | && size.height() <= constraints->max_height; |
368 | } |
369 | |
370 | AVHWDeviceType HWAccel::deviceType() const |
371 | { |
372 | return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE; |
373 | } |
374 | |
375 | void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size) |
376 | { |
377 | if (m_hwFramesContext) { |
378 | qWarning() << "Frames context has been already created!" ; |
379 | return; |
380 | } |
381 | |
382 | if (!m_hwDeviceContext) |
383 | return; |
384 | |
385 | m_hwFramesContext.reset(p: av_hwframe_ctx_alloc(device_ctx: m_hwDeviceContext.get())); |
386 | auto *c = (AVHWFramesContext *)m_hwFramesContext->data; |
387 | c->format = hwFormat(); |
388 | c->sw_format = swFormat; |
389 | c->width = size.width(); |
390 | c->height = size.height(); |
391 | qCDebug(qLHWAccel) << "init frames context" ; |
392 | int err = av_hwframe_ctx_init(ref: m_hwFramesContext.get()); |
393 | if (err < 0) |
394 | qWarning() << "failed to init HW frame context" << err << err2str(errnum: err); |
395 | else |
396 | qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format; |
397 | } |
398 | |
399 | AVHWFramesContext *HWAccel::hwFramesContext() const |
400 | { |
401 | return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr; |
402 | } |
403 | |
404 | static void deleteHwFrameContextData(AVHWFramesContext *context) |
405 | { |
406 | delete reinterpret_cast<HwFrameContextData *>(context->user_opaque); |
407 | } |
408 | |
409 | HwFrameContextData &HwFrameContextData::ensure(AVFrame &hwFrame) |
410 | { |
411 | Q_ASSERT(hwFrame.hw_frames_ctx && hwFrame.hw_frames_ctx->data); |
412 | |
413 | auto context = reinterpret_cast<AVHWFramesContext *>(hwFrame.hw_frames_ctx->data); |
414 | if (!context->user_opaque) { |
415 | context->user_opaque = new HwFrameContextData; |
416 | Q_ASSERT(!context->free); |
417 | context->free = deleteHwFrameContextData; |
418 | } else { |
419 | Q_ASSERT(context->free == deleteHwFrameContextData); |
420 | } |
421 | |
422 | return *reinterpret_cast<HwFrameContextData *>(context->user_opaque); |
423 | } |
424 | |
425 | AVFrameUPtr copyFromHwPool(AVFrameUPtr frame) |
426 | { |
427 | #if QT_CONFIG(wmf) |
428 | return copyFromHwPoolD3D11(std::move(frame)); |
429 | #else |
430 | return frame; |
431 | #endif |
432 | } |
433 | |
434 | } // namespace QFFmpeg |
435 | |
436 | QT_END_NAMESPACE |
437 | |