1 | // Copyright (C) 2021 The Qt Company Ltd. |
2 | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | |
4 | #include <common/qglist_helper_p.h> |
5 | #include "qgstreamerformatinfo_p.h" |
6 | |
7 | #include <gst/gst.h> |
8 | |
9 | QT_BEGIN_NAMESPACE |
10 | |
11 | QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructureView structure) |
12 | { |
13 | using namespace std::string_view_literals; |
14 | const char *name = structure.name().data(); |
15 | |
16 | if (!name || (strncmp(s1: name, s2: "audio/" , n: 6) != 0)) |
17 | return QMediaFormat::AudioCodec::Unspecified; |
18 | name += 6; |
19 | if (name == "mpeg"sv ) { |
20 | auto version = structure["mpegversion" ].toInt(); |
21 | if (version == 1) { |
22 | auto layer = structure["layer" ]; |
23 | if (!layer.isNull()) |
24 | return QMediaFormat::AudioCodec::MP3; |
25 | } |
26 | if (version == 4) |
27 | return QMediaFormat::AudioCodec::AAC; |
28 | return QMediaFormat::AudioCodec::Unspecified; |
29 | } |
30 | if (name == "x-ac3"sv ) |
31 | return QMediaFormat::AudioCodec::AC3; |
32 | |
33 | if (name == "x-eac3"sv ) |
34 | return QMediaFormat::AudioCodec::EAC3; |
35 | |
36 | if (name == "x-flac"sv ) |
37 | return QMediaFormat::AudioCodec::FLAC; |
38 | |
39 | if (name == "x-alac"sv ) |
40 | return QMediaFormat::AudioCodec::ALAC; |
41 | |
42 | if (name == "x-true-hd"sv ) |
43 | return QMediaFormat::AudioCodec::DolbyTrueHD; |
44 | |
45 | if (name == "x-vorbis"sv ) |
46 | return QMediaFormat::AudioCodec::Vorbis; |
47 | |
48 | if (name == "x-opus"sv ) |
49 | return QMediaFormat::AudioCodec::Opus; |
50 | |
51 | if (name == "x-wav"sv ) |
52 | return QMediaFormat::AudioCodec::Wave; |
53 | |
54 | if (name == "x-wma"sv ) |
55 | return QMediaFormat::AudioCodec::WMA; |
56 | |
57 | return QMediaFormat::AudioCodec::Unspecified; |
58 | } |
59 | |
60 | QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructureView structure) |
61 | { |
62 | using namespace std::string_view_literals; |
63 | const char *name = structure.name().data(); |
64 | |
65 | if (!name || (strncmp(s1: name, s2: "video/" , n: 6) != 0)) |
66 | return QMediaFormat::VideoCodec::Unspecified; |
67 | name += 6; |
68 | |
69 | if (name == "mpeg"sv ) { |
70 | auto version = structure["mpegversion" ].toInt(); |
71 | if (version == 1) |
72 | return QMediaFormat::VideoCodec::MPEG1; |
73 | if (version == 2) |
74 | return QMediaFormat::VideoCodec::MPEG2; |
75 | if (version == 4) |
76 | return QMediaFormat::VideoCodec::MPEG4; |
77 | return QMediaFormat::VideoCodec::Unspecified; |
78 | } |
79 | if (name == "x-h264"sv ) |
80 | return QMediaFormat::VideoCodec::H264; |
81 | |
82 | if (name == "x-h265"sv ) |
83 | return QMediaFormat::VideoCodec::H265; |
84 | |
85 | if (name == "x-vp8"sv ) |
86 | return QMediaFormat::VideoCodec::VP8; |
87 | |
88 | if (name == "x-vp9"sv ) |
89 | return QMediaFormat::VideoCodec::VP9; |
90 | |
91 | if (name == "x-av1"sv ) |
92 | return QMediaFormat::VideoCodec::AV1; |
93 | |
94 | if (name == "x-theora"sv ) |
95 | return QMediaFormat::VideoCodec::Theora; |
96 | |
97 | if (name == "x-jpeg"sv ) |
98 | return QMediaFormat::VideoCodec::MotionJPEG; |
99 | |
100 | if (name == "x-wmv"sv ) |
101 | return QMediaFormat::VideoCodec::WMV; |
102 | |
103 | return QMediaFormat::VideoCodec::Unspecified; |
104 | } |
105 | |
106 | QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructureView structure) |
107 | { |
108 | using namespace std::string_view_literals; |
109 | const char *name = structure.name().data(); |
110 | |
111 | if (name == "video/x-ms-asf"sv ) |
112 | return QMediaFormat::FileFormat::WMV; |
113 | |
114 | if (name == "video/x-msvideo"sv ) |
115 | return QMediaFormat::FileFormat::AVI; |
116 | |
117 | if (name == "video/x-matroska"sv ) |
118 | return QMediaFormat::FileFormat::Matroska; |
119 | |
120 | if (name == "video/quicktime"sv ) { |
121 | const char *variant = structure["variant" ].toString(); |
122 | if (!variant) |
123 | return QMediaFormat::FileFormat::QuickTime; |
124 | if (variant == "iso"sv ) |
125 | return QMediaFormat::FileFormat::MPEG4; |
126 | } |
127 | if (name == "video/ogg"sv ) |
128 | return QMediaFormat::FileFormat::Ogg; |
129 | |
130 | if (name == "video/webm"sv ) |
131 | return QMediaFormat::FileFormat::WebM; |
132 | |
133 | if (name == "audio/x-m4a"sv ) |
134 | return QMediaFormat::FileFormat::Mpeg4Audio; |
135 | |
136 | if (name == "audio/x-wav"sv ) |
137 | return QMediaFormat::FileFormat::Wave; |
138 | |
139 | if (name == "audio/mpeg"sv ) { |
140 | auto mpegversion = structure["mpegversion" ].toInt(); |
141 | if (mpegversion == 1) { |
142 | auto layer = structure["layer" ]; |
143 | if (!layer.isNull()) |
144 | return QMediaFormat::FileFormat::MP3; |
145 | } |
146 | } |
147 | |
148 | if (name == "audio/aac"sv ) |
149 | return QMediaFormat::FileFormat::AAC; |
150 | |
151 | if (name == "audio/x-ms-wma"sv ) |
152 | return QMediaFormat::FileFormat::WMA; |
153 | |
154 | if (name == "audio/x-flac"sv ) |
155 | return QMediaFormat::FileFormat::FLAC; |
156 | |
157 | return QMediaFormat::UnspecifiedFormat; |
158 | } |
159 | |
160 | |
161 | QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructureView structure) |
162 | { |
163 | using namespace std::string_view_literals; |
164 | const char *name = structure.name().data(); |
165 | |
166 | if (name == "image/jpeg"sv ) |
167 | return QImageCapture::JPEG; |
168 | |
169 | if (name == "image/png"sv ) |
170 | return QImageCapture::PNG; |
171 | |
172 | if (name == "image/webp"sv ) |
173 | return QImageCapture::WebP; |
174 | |
175 | if (name == "image/tiff"sv ) |
176 | return QImageCapture::Tiff; |
177 | |
178 | return QImageCapture::UnspecifiedFormat; |
179 | } |
180 | |
181 | static QPair<QList<QMediaFormat::AudioCodec>, QList<QMediaFormat::VideoCodec>> getCodecsList(bool decode) |
182 | { |
183 | QList<QMediaFormat::AudioCodec> audio; |
184 | QList<QMediaFormat::VideoCodec> video; |
185 | |
186 | GstPadDirection padDirection = decode ? GST_PAD_SINK : GST_PAD_SRC; |
187 | |
188 | GList *elementList = gst_element_factory_list_get_elements(type: decode ? GST_ELEMENT_FACTORY_TYPE_DECODER : GST_ELEMENT_FACTORY_TYPE_ENCODER, |
189 | minrank: GST_RANK_MARGINAL); |
190 | |
191 | for (GstElementFactory *factory : |
192 | QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { |
193 | for (GstStaticPadTemplate *padTemplate : |
194 | QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>( |
195 | gst_element_factory_get_static_pad_templates(factory))) { |
196 | if (padTemplate->direction == padDirection) { |
197 | auto caps = QGstCaps(gst_static_caps_get(static_caps: &padTemplate->static_caps), QGstCaps::HasRef); |
198 | |
199 | for (int i = 0; i < caps.size(); i++) { |
200 | QGstStructureView structure = caps.at(index: i); |
201 | auto a = QGstreamerFormatInfo::audioCodecForCaps(structure); |
202 | if (a != QMediaFormat::AudioCodec::Unspecified && !audio.contains(t: a)) |
203 | audio.append(t: a); |
204 | auto v = QGstreamerFormatInfo::videoCodecForCaps(structure); |
205 | if (v != QMediaFormat::VideoCodec::Unspecified && !video.contains(t: v)) |
206 | video.append(t: v); |
207 | } |
208 | } |
209 | } |
210 | } |
211 | gst_plugin_feature_list_free(list: elementList); |
212 | return {audio, video}; |
213 | } |
214 | |
215 | QList<QGstreamerFormatInfo::CodecMap> |
216 | QGstreamerFormatInfo::getCodecMaps(QMediaFormat::ConversionMode conversionMode, |
217 | QList<QMediaFormat::AudioCodec> supportedAudioCodecs, |
218 | QList<QMediaFormat::VideoCodec> supportedVideoCodecs) |
219 | { |
220 | QList<QGstreamerFormatInfo::CodecMap> maps; |
221 | |
222 | GstPadDirection dataPadDirection = |
223 | (conversionMode == QMediaFormat::Decode) ? GST_PAD_SINK : GST_PAD_SRC; |
224 | |
225 | auto encodableFactoryTypes = |
226 | (GST_ELEMENT_FACTORY_TYPE_MUXER | GST_ELEMENT_FACTORY_TYPE_PAYLOADER |
227 | | GST_ELEMENT_FACTORY_TYPE_ENCRYPTOR | GST_ELEMENT_FACTORY_TYPE_ENCODER); |
228 | GList *elementList = gst_element_factory_list_get_elements( |
229 | type: (conversionMode == QMediaFormat::Decode) ? GST_ELEMENT_FACTORY_TYPE_DECODABLE |
230 | : encodableFactoryTypes, |
231 | minrank: GST_RANK_MARGINAL); |
232 | |
233 | for (GstElementFactory *factory : |
234 | QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { |
235 | QList<QMediaFormat::FileFormat> fileFormats; |
236 | |
237 | for (GstStaticPadTemplate *padTemplate : |
238 | QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>( |
239 | gst_element_factory_get_static_pad_templates(factory))) { |
240 | |
241 | // Check pads on data side for file formats, except for parsers check source side |
242 | if (padTemplate->direction == dataPadDirection |
243 | || (gst_element_factory_list_is_type(factory, GST_ELEMENT_FACTORY_TYPE_PARSER) |
244 | && padTemplate->direction == GST_PAD_SRC)) { |
245 | auto caps = QGstCaps(gst_static_caps_get(static_caps: &padTemplate->static_caps), QGstCaps::HasRef); |
246 | |
247 | for (int i = 0; i < caps.size(); i++) { |
248 | QGstStructureView structure = caps.at(index: i); |
249 | auto fmt = fileFormatForCaps(structure); |
250 | if (fmt != QMediaFormat::UnspecifiedFormat) |
251 | fileFormats.append(t: fmt); |
252 | } |
253 | } |
254 | } |
255 | |
256 | if (fileFormats.isEmpty()) |
257 | continue; |
258 | |
259 | QList<QMediaFormat::AudioCodec> audioCodecs; |
260 | QList<QMediaFormat::VideoCodec> videoCodecs; |
261 | |
262 | for (GstStaticPadTemplate *padTemplate : |
263 | QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>( |
264 | gst_element_factory_get_static_pad_templates(factory))) { |
265 | |
266 | // check the other side for supported inputs/outputs |
267 | if (padTemplate->direction != dataPadDirection) { |
268 | auto caps = QGstCaps(gst_static_caps_get(static_caps: &padTemplate->static_caps), QGstCaps::HasRef); |
269 | |
270 | bool acceptsRawAudio = false; |
271 | for (int i = 0; i < caps.size(); i++) { |
272 | QGstStructureView structure = caps.at(index: i); |
273 | if (structure.name() == "audio/x-raw" ) |
274 | acceptsRawAudio = true; |
275 | auto audio = audioCodecForCaps(structure); |
276 | if (audio != QMediaFormat::AudioCodec::Unspecified && supportedAudioCodecs.contains(t: audio)) |
277 | audioCodecs.append(t: audio); |
278 | auto video = videoCodecForCaps(structure); |
279 | if (video != QMediaFormat::VideoCodec::Unspecified && supportedVideoCodecs.contains(t: video)) |
280 | videoCodecs.append(t: video); |
281 | } |
282 | if (acceptsRawAudio && fileFormats.size() == 1) { |
283 | switch (fileFormats.at(i: 0)) { |
284 | case QMediaFormat::Mpeg4Audio: |
285 | default: |
286 | break; |
287 | case QMediaFormat::MP3: |
288 | audioCodecs.append(t: QMediaFormat::AudioCodec::MP3); |
289 | break; |
290 | case QMediaFormat::FLAC: |
291 | audioCodecs.append(t: QMediaFormat::AudioCodec::FLAC); |
292 | break; |
293 | case QMediaFormat::Wave: |
294 | audioCodecs.append(t: QMediaFormat::AudioCodec::Wave); |
295 | break; |
296 | } |
297 | } |
298 | } |
299 | } |
300 | if (!audioCodecs.isEmpty() || !videoCodecs.isEmpty()) { |
301 | for (auto f : std::as_const(t&: fileFormats)) { |
302 | maps.append(t: {.format: f, .audio: audioCodecs, .video: videoCodecs}); |
303 | if (f == QMediaFormat::MPEG4 && !fileFormats.contains(t: QMediaFormat::Mpeg4Audio)) { |
304 | maps.append(t: {.format: QMediaFormat::Mpeg4Audio, .audio: audioCodecs, .video: {}}); |
305 | if (audioCodecs.contains(t: QMediaFormat::AudioCodec::AAC)) |
306 | maps.append(t: {.format: QMediaFormat::AAC, .audio: { QMediaFormat::AudioCodec::AAC }, .video: {}}); |
307 | } else if (f == QMediaFormat::WMV && !fileFormats.contains(t: QMediaFormat::WMA)) { |
308 | maps.append(t: {.format: QMediaFormat::WMA, .audio: audioCodecs, .video: {}}); |
309 | } |
310 | } |
311 | } |
312 | } |
313 | gst_plugin_feature_list_free(list: elementList); |
314 | return maps; |
315 | } |
316 | |
317 | static QList<QImageCapture::FileFormat> getImageFormatList() |
318 | { |
319 | QSet<QImageCapture::FileFormat> formats; |
320 | |
321 | GList *elementList = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER, |
322 | minrank: GST_RANK_MARGINAL); |
323 | |
324 | for (GstElementFactory *factory : |
325 | QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { |
326 | |
327 | for (GstStaticPadTemplate *padTemplate : |
328 | QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>( |
329 | gst_element_factory_get_static_pad_templates(factory))) { |
330 | if (padTemplate->direction == GST_PAD_SRC) { |
331 | QGstCaps caps = QGstCaps(gst_static_caps_get(static_caps: &padTemplate->static_caps), QGstCaps::HasRef); |
332 | |
333 | for (int i = 0; i < caps.size(); i++) { |
334 | QGstStructureView structure = caps.at(index: i); |
335 | auto f = QGstreamerFormatInfo::imageFormatForCaps(structure); |
336 | if (f != QImageCapture::UnspecifiedFormat) { |
337 | // qDebug() << structure.toString() << f; |
338 | formats.insert(value: f); |
339 | } |
340 | } |
341 | } |
342 | } |
343 | } |
344 | gst_plugin_feature_list_free(list: elementList); |
345 | return formats.values(); |
346 | } |
347 | |
348 | #if 0 |
349 | static void dumpAudioCodecs(const QList<QMediaFormat::AudioCodec> &codecList) |
350 | { |
351 | qDebug() << "Audio codecs:" ; |
352 | for (const auto &c : codecList) |
353 | qDebug() << " " << QMediaFormat::audioCodecName(c); |
354 | } |
355 | |
356 | static void dumpVideoCodecs(const QList<QMediaFormat::VideoCodec> &codecList) |
357 | { |
358 | qDebug() << "Video codecs:" ; |
359 | for (const auto &c : codecList) |
360 | qDebug() << " " << QMediaFormat::videoCodecName(c); |
361 | } |
362 | |
363 | static void dumpMuxers(const QList<QPlatformMediaFormatInfo::CodecMap> &muxerList) |
364 | { |
365 | for (const auto &m : muxerList) { |
366 | qDebug() << " " << QMediaFormat::fileFormatName(m.format); |
367 | qDebug() << " Audio" ; |
368 | for (const auto &a : m.audio) |
369 | qDebug() << " " << QMediaFormat::audioCodecName(a); |
370 | qDebug() << " Video" ; |
371 | for (const auto &v : m.video) |
372 | qDebug() << " " << QMediaFormat::videoCodecName(v); |
373 | } |
374 | |
375 | } |
376 | #endif |
377 | |
378 | QGstreamerFormatInfo::QGstreamerFormatInfo() |
379 | { |
380 | auto codecs = getCodecsList(/*decode = */ true); |
381 | decoders = getCodecMaps(conversionMode: QMediaFormat::Decode, supportedAudioCodecs: codecs.first, supportedVideoCodecs: codecs.second); |
382 | |
383 | codecs = getCodecsList(/*decode = */ false); |
384 | encoders = getCodecMaps(conversionMode: QMediaFormat::Encode, supportedAudioCodecs: codecs.first, supportedVideoCodecs: codecs.second); |
385 | // dumpAudioCodecs(codecs.first); |
386 | // dumpVideoCodecs(codecs.second); |
387 | // dumpMuxers(encoders); |
388 | |
389 | imageFormats = getImageFormatList(); |
390 | } |
391 | |
392 | QGstreamerFormatInfo::~QGstreamerFormatInfo() = default; |
393 | |
394 | QGstCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const |
395 | { |
396 | auto format = f.fileFormat(); |
397 | Q_ASSERT(format != QMediaFormat::UnspecifiedFormat); |
398 | |
399 | const char *capsForFormat[QMediaFormat::LastFileFormat + 1] = { |
400 | "video/x-ms-asf" , // WMV |
401 | "video/x-msvideo" , // AVI |
402 | "video/x-matroska" , // Matroska |
403 | "video/quicktime, variant=(string)iso" , // MPEG4 |
404 | "video/ogg" , // Ogg |
405 | "video/quicktime" , // QuickTime |
406 | "video/webm" , // WebM |
407 | "video/quicktime, variant=(string)iso" , // Mpeg4Audio is the same is mp4... |
408 | "video/quicktime, variant=(string)iso" , // AAC is also an MP4 container |
409 | "video/x-ms-asf" , // WMA, same as WMV |
410 | "audio/mpeg, mpegversion=(int)1, layer=(int)3" , // MP3 |
411 | "audio/x-flac" , // FLAC |
412 | "audio/x-wav" // Wave |
413 | }; |
414 | return QGstCaps(gst_caps_from_string(string: capsForFormat[format]), QGstCaps::HasRef); |
415 | } |
416 | |
417 | QGstCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const |
418 | { |
419 | auto codec = f.audioCodec(); |
420 | if (codec == QMediaFormat::AudioCodec::Unspecified) |
421 | return {}; |
422 | |
423 | const char *capsForCodec[(int)QMediaFormat::AudioCodec::LastAudioCodec + 1] = { |
424 | "audio/mpeg, mpegversion=(int)1, layer=(int)3" , // MP3 |
425 | "audio/mpeg, mpegversion=(int)4" , // AAC |
426 | "audio/x-ac3" , // AC3 |
427 | "audio/x-eac3" , // EAC3 |
428 | "audio/x-flac" , // FLAC |
429 | "audio/x-true-hd" , // DolbyTrueHD |
430 | "audio/x-opus" , // Opus |
431 | "audio/x-vorbis" , // Vorbis |
432 | "audio/x-raw" , // WAVE |
433 | "audio/x-wma" , // WMA |
434 | "audio/x-alac" , // ALAC |
435 | }; |
436 | return QGstCaps(gst_caps_from_string(string: capsForCodec[(int)codec]), QGstCaps::HasRef); |
437 | } |
438 | |
439 | QGstCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const |
440 | { |
441 | auto codec = f.videoCodec(); |
442 | if (codec == QMediaFormat::VideoCodec::Unspecified) |
443 | return {}; |
444 | |
445 | const char *capsForCodec[(int)QMediaFormat::VideoCodec::LastVideoCodec + 1] = { |
446 | "video/mpeg, mpegversion=(int)1" , // MPEG1, |
447 | "video/mpeg, mpegversion=(int)2" , // MPEG2, |
448 | "video/mpeg, mpegversion=(int)4" , // MPEG4, |
449 | "video/x-h264" , // H264, |
450 | "video/x-h265" , // H265, |
451 | "video/x-vp8" , // VP8, |
452 | "video/x-vp9" , // VP9, |
453 | "video/x-av1" , // AV1, |
454 | "video/x-theora" , // Theora, |
455 | "audio/x-wmv" , // WMV |
456 | "video/x-jpeg" , // MotionJPEG, |
457 | }; |
458 | return QGstCaps(gst_caps_from_string(string: capsForCodec[(int)codec]), QGstCaps::HasRef); |
459 | } |
460 | |
461 | QT_END_NAMESPACE |
462 | |