1// This file is part of OpenCV project.
2// It is subject to the license terms in the LICENSE file found in the top-level directory
3// of this distribution and at http://opencv.org/license.html.
4//
5// Copyright (C) 2021 Intel Corporation
6#ifdef HAVE_ONEVPL
7#include <errno.h>
8
9#include <opencv2/gapi/own/assert.hpp>
10#include "streaming/onevpl/demux/async_mfp_demux_data_provider.hpp"
11#include "logger.hpp"
12
13namespace cv {
14namespace gapi {
15namespace wip {
16namespace onevpl {
17#ifdef _WIN32
18static HRESULT create_media_source(const std::string& url, IMFMediaSource **ppSource) {
19 wchar_t sURL[MAX_PATH];
20 GAPI_Assert(url.size() < MAX_PATH && "Windows MAX_PATH limit was reached");
21 size_t ret_url_length = 0;
22 mbstowcs_s(&ret_url_length, sURL, url.data(), url.size());
23
24 HRESULT hr = S_OK;
25 ComPtrGuard<IMFSourceResolver> source_resolver = createCOMPtrGuard<IMFSourceResolver>();
26 {
27 IMFSourceResolver *source_resolver_tmp = nullptr;
28 hr = MFCreateSourceResolver(&source_resolver_tmp);
29 if (FAILED(hr)) {
30 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
31 "cannot create MFCreateSourceResolver from URI: " +
32 url);
33 }
34 source_resolver.reset(source_resolver_tmp);
35 }
36
37 MF_OBJECT_TYPE ObjectType = MF_OBJECT_INVALID;
38 /**
39 * NB:
40 * CreateObjectFromURL throws exception if actual container type is mismatched with
41 * file extension. To overcome this situation by MFP it is possible to apply 2 step
42 * approach: at first step we pass special flag
43 * `MF_RESOLUTION_KEEP_BYTE_STREAM_ALIVE_ON_FAIL` which claims to fail with error
44 * in any case of input instead exception throwing;
45 * at the second step we must cease `MF_RESOLUTION_KEEP_BYTE_STREAM_ALIVE_ON_FAIL`
46 * flag AND set another special flag
47 * `MF_RESOLUTION_CONTENT_DOES_NOT_HAVE_TO_MATCH_EXTENSION_OR_MIME_TYPE`
48 * to filter out container type & file extension mismatch errors.
49 *
50 * If it failed at second phase then some other errors were not related
51 * to types-extension disturbance would happen and data provider must fail ultimately.
52 *
53 * If second step passed then data provider would continue execution
54 */
55 IUnknown *source_unknown_tmp = nullptr;
56 DWORD resolver_flags = MF_RESOLUTION_MEDIASOURCE | MF_RESOLUTION_READ |
57 MF_RESOLUTION_KEEP_BYTE_STREAM_ALIVE_ON_FAIL;
58 hr = source_resolver->CreateObjectFromURL(sURL,
59 resolver_flags,
60 nullptr, &ObjectType,
61 &source_unknown_tmp);
62 if (FAILED(hr)) {
63 GAPI_LOG_DEBUG(nullptr, "Cannot create MF_RESOLUTION_MEDIASOURCE using file extension, "
64 " looks like actual media container type doesn't match to file extension. "
65 "Try special mode");
66 resolver_flags ^= MF_RESOLUTION_KEEP_BYTE_STREAM_ALIVE_ON_FAIL;
67 resolver_flags ^= MF_RESOLUTION_CONTENT_DOES_NOT_HAVE_TO_MATCH_EXTENSION_OR_MIME_TYPE;
68 hr = source_resolver->CreateObjectFromURL(sURL, resolver_flags,
69 nullptr, &ObjectType,
70 &source_unknown_tmp);
71 if (FAILED(hr)) {
72 GAPI_LOG_WARNING(nullptr, "Cannot create MF_RESOLUTION_MEDIASOURCE from URI: " <<
73 url << ". Abort");
74 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
75 "CreateObjectFromURL failed");
76 }
77 }
78
79 ComPtrGuard<IUnknown> source_unknown = createCOMPtrGuard(source_unknown_tmp);
80 hr = source_unknown->QueryInterface(__uuidof(IMFMediaSource), (void**)ppSource);
81 if (FAILED(hr)) {
82 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
83 "QueryInterface for IMFMediaSource failed");
84 }
85
86 return hr;
87}
88
89/*
90 * The next part of converting GUID into string function GetGUIDNameConst
91 * was copied and modified from
92 * https://docs.microsoft.com/en-us/windows/win32/medfound/media-type-debugging-code
93 */
94#ifndef IF_EQUAL_RETURN
95#define IF_EQUAL_RETURN(param, val) if(val == param) return #val
96#endif
97
98static const char* GetGUIDNameConst(const GUID& guid)
99{
100 IF_EQUAL_RETURN(guid, MF_MT_MAJOR_TYPE);
101 IF_EQUAL_RETURN(guid, MF_MT_MAJOR_TYPE);
102 IF_EQUAL_RETURN(guid, MF_MT_SUBTYPE);
103 IF_EQUAL_RETURN(guid, MF_MT_ALL_SAMPLES_INDEPENDENT);
104 IF_EQUAL_RETURN(guid, MF_MT_FIXED_SIZE_SAMPLES);
105 IF_EQUAL_RETURN(guid, MF_MT_COMPRESSED);
106 IF_EQUAL_RETURN(guid, MF_MT_SAMPLE_SIZE);
107 IF_EQUAL_RETURN(guid, MF_MT_WRAPPED_TYPE);
108 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_NUM_CHANNELS);
109 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_SAMPLES_PER_SECOND);
110 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_FLOAT_SAMPLES_PER_SECOND);
111 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_AVG_BYTES_PER_SECOND);
112 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_BLOCK_ALIGNMENT);
113 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_BITS_PER_SAMPLE);
114 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_VALID_BITS_PER_SAMPLE);
115 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_SAMPLES_PER_BLOCK);
116 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_CHANNEL_MASK);
117 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_FOLDDOWN_MATRIX);
118 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_WMADRC_PEAKREF);
119 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_WMADRC_PEAKTARGET);
120 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_WMADRC_AVGREF);
121 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_WMADRC_AVGTARGET);
122 IF_EQUAL_RETURN(guid, MF_MT_AUDIO_PREFER_WAVEFORMATEX);
123 IF_EQUAL_RETURN(guid, MF_MT_AAC_PAYLOAD_TYPE);
124 IF_EQUAL_RETURN(guid, MF_MT_AAC_AUDIO_PROFILE_LEVEL_INDICATION);
125 IF_EQUAL_RETURN(guid, MF_MT_FRAME_SIZE);
126 IF_EQUAL_RETURN(guid, MF_MT_FRAME_RATE);
127 IF_EQUAL_RETURN(guid, MF_MT_FRAME_RATE_RANGE_MAX);
128 IF_EQUAL_RETURN(guid, MF_MT_FRAME_RATE_RANGE_MIN);
129 IF_EQUAL_RETURN(guid, MF_MT_PIXEL_ASPECT_RATIO);
130 IF_EQUAL_RETURN(guid, MF_MT_DRM_FLAGS);
131 IF_EQUAL_RETURN(guid, MF_MT_PAD_CONTROL_FLAGS);
132 IF_EQUAL_RETURN(guid, MF_MT_SOURCE_CONTENT_HINT);
133 IF_EQUAL_RETURN(guid, MF_MT_VIDEO_CHROMA_SITING);
134 IF_EQUAL_RETURN(guid, MF_MT_INTERLACE_MODE);
135 IF_EQUAL_RETURN(guid, MF_MT_TRANSFER_FUNCTION);
136 IF_EQUAL_RETURN(guid, MF_MT_VIDEO_PRIMARIES);
137 IF_EQUAL_RETURN(guid, MF_MT_CUSTOM_VIDEO_PRIMARIES);
138 IF_EQUAL_RETURN(guid, MF_MT_YUV_MATRIX);
139 IF_EQUAL_RETURN(guid, MF_MT_VIDEO_LIGHTING);
140 IF_EQUAL_RETURN(guid, MF_MT_VIDEO_NOMINAL_RANGE);
141 IF_EQUAL_RETURN(guid, MF_MT_GEOMETRIC_APERTURE);
142 IF_EQUAL_RETURN(guid, MF_MT_MINIMUM_DISPLAY_APERTURE);
143 IF_EQUAL_RETURN(guid, MF_MT_PAN_SCAN_APERTURE);
144 IF_EQUAL_RETURN(guid, MF_MT_PAN_SCAN_ENABLED);
145 IF_EQUAL_RETURN(guid, MF_MT_AVG_BITRATE);
146 IF_EQUAL_RETURN(guid, MF_MT_AVG_BIT_ERROR_RATE);
147 IF_EQUAL_RETURN(guid, MF_MT_MAX_KEYFRAME_SPACING);
148 IF_EQUAL_RETURN(guid, MF_MT_DEFAULT_STRIDE);
149 IF_EQUAL_RETURN(guid, MF_MT_PALETTE);
150 IF_EQUAL_RETURN(guid, MF_MT_USER_DATA);
151 IF_EQUAL_RETURN(guid, MF_MT_AM_FORMAT_TYPE);
152 IF_EQUAL_RETURN(guid, MF_MT_MPEG_START_TIME_CODE);
153 IF_EQUAL_RETURN(guid, MF_MT_MPEG2_PROFILE);
154 IF_EQUAL_RETURN(guid, MF_MT_MPEG2_LEVEL);
155 IF_EQUAL_RETURN(guid, MF_MT_MPEG2_FLAGS);
156 IF_EQUAL_RETURN(guid, MF_MT_MPEG_SEQUENCE_HEADER);
157 IF_EQUAL_RETURN(guid, MF_MT_DV_AAUX_SRC_PACK_0);
158 IF_EQUAL_RETURN(guid, MF_MT_DV_AAUX_CTRL_PACK_0);
159 IF_EQUAL_RETURN(guid, MF_MT_DV_AAUX_SRC_PACK_1);
160 IF_EQUAL_RETURN(guid, MF_MT_DV_AAUX_CTRL_PACK_1);
161 IF_EQUAL_RETURN(guid, MF_MT_DV_VAUX_SRC_PACK);
162 IF_EQUAL_RETURN(guid, MF_MT_DV_VAUX_CTRL_PACK);
163 IF_EQUAL_RETURN(guid, MF_MT_ARBITRARY_HEADER);
164 IF_EQUAL_RETURN(guid, MF_MT_ARBITRARY_FORMAT);
165 IF_EQUAL_RETURN(guid, MF_MT_IMAGE_LOSS_TOLERANT);
166 IF_EQUAL_RETURN(guid, MF_MT_MPEG4_SAMPLE_DESCRIPTION);
167 IF_EQUAL_RETURN(guid, MF_MT_MPEG4_CURRENT_SAMPLE_ENTRY);
168 IF_EQUAL_RETURN(guid, MF_MT_ORIGINAL_4CC);
169 IF_EQUAL_RETURN(guid, MF_MT_ORIGINAL_WAVE_FORMAT_TAG);
170
171 // Media types
172
173 IF_EQUAL_RETURN(guid, MFMediaType_Audio);
174 IF_EQUAL_RETURN(guid, MFMediaType_Video);
175 IF_EQUAL_RETURN(guid, MFMediaType_Protected);
176 IF_EQUAL_RETURN(guid, MFMediaType_SAMI);
177 IF_EQUAL_RETURN(guid, MFMediaType_Script);
178 IF_EQUAL_RETURN(guid, MFMediaType_Image);
179 IF_EQUAL_RETURN(guid, MFMediaType_HTML);
180 IF_EQUAL_RETURN(guid, MFMediaType_Binary);
181 IF_EQUAL_RETURN(guid, MFMediaType_FileTransfer);
182
183 IF_EQUAL_RETURN(guid, MFVideoFormat_AI44); // FCC('AI44')
184 IF_EQUAL_RETURN(guid, MFVideoFormat_ARGB32); // D3DFMT_A8R8G8B8
185 IF_EQUAL_RETURN(guid, MFVideoFormat_AV1);
186 IF_EQUAL_RETURN(guid, MFVideoFormat_AYUV); // FCC('AYUV')
187 IF_EQUAL_RETURN(guid, MFVideoFormat_DV25); // FCC('dv25')
188 IF_EQUAL_RETURN(guid, MFVideoFormat_DV50); // FCC('dv50')
189 IF_EQUAL_RETURN(guid, MFVideoFormat_DVH1); // FCC('dvh1')
190 IF_EQUAL_RETURN(guid, MFVideoFormat_DVSD); // FCC('dvsd')
191 IF_EQUAL_RETURN(guid, MFVideoFormat_DVSL); // FCC('dvsl')
192 IF_EQUAL_RETURN(guid, MFVideoFormat_H264); // FCC('H264')
193 IF_EQUAL_RETURN(guid, MFVideoFormat_H265);
194 IF_EQUAL_RETURN(guid, MFVideoFormat_HEVC);
195 IF_EQUAL_RETURN(guid, MFVideoFormat_HEVC_ES);
196 IF_EQUAL_RETURN(guid, MFVideoFormat_I420); // FCC('I420')
197 IF_EQUAL_RETURN(guid, MFVideoFormat_IYUV); // FCC('IYUV')
198 IF_EQUAL_RETURN(guid, MFVideoFormat_M4S2); // FCC('M4S2')
199 IF_EQUAL_RETURN(guid, MFVideoFormat_MJPG);
200 IF_EQUAL_RETURN(guid, MFVideoFormat_MP43); // FCC('MP43')
201 IF_EQUAL_RETURN(guid, MFVideoFormat_MP4S); // FCC('MP4S')
202 IF_EQUAL_RETURN(guid, MFVideoFormat_MP4V); // FCC('MP4V')
203 IF_EQUAL_RETURN(guid, MFVideoFormat_MPG1); // FCC('MPG1')
204 IF_EQUAL_RETURN(guid, MFVideoFormat_MSS1); // FCC('MSS1')
205 IF_EQUAL_RETURN(guid, MFVideoFormat_MSS2); // FCC('MSS2')
206 IF_EQUAL_RETURN(guid, MFVideoFormat_NV11); // FCC('NV11')
207 IF_EQUAL_RETURN(guid, MFVideoFormat_NV12); // FCC('NV12')
208 IF_EQUAL_RETURN(guid, MFVideoFormat_P010); // FCC('P010')
209 IF_EQUAL_RETURN(guid, MFVideoFormat_P016); // FCC('P016')
210 IF_EQUAL_RETURN(guid, MFVideoFormat_P210); // FCC('P210')
211 IF_EQUAL_RETURN(guid, MFVideoFormat_P216); // FCC('P216')
212 IF_EQUAL_RETURN(guid, MFVideoFormat_RGB24); // D3DFMT_R8G8B8
213 IF_EQUAL_RETURN(guid, MFVideoFormat_RGB32); // D3DFMT_X8R8G8B8
214 IF_EQUAL_RETURN(guid, MFVideoFormat_RGB555); // D3DFMT_X1R5G5B5
215 IF_EQUAL_RETURN(guid, MFVideoFormat_RGB565); // D3DFMT_R5G6B5
216 IF_EQUAL_RETURN(guid, MFVideoFormat_RGB8);
217 IF_EQUAL_RETURN(guid, MFVideoFormat_UYVY); // FCC('UYVY')
218 IF_EQUAL_RETURN(guid, MFVideoFormat_v210); // FCC('v210')
219 IF_EQUAL_RETURN(guid, MFVideoFormat_v410); // FCC('v410')
220 IF_EQUAL_RETURN(guid, MFVideoFormat_WMV1); // FCC('WMV1')
221 IF_EQUAL_RETURN(guid, MFVideoFormat_WMV2); // FCC('WMV2')
222 IF_EQUAL_RETURN(guid, MFVideoFormat_WMV3); // FCC('WMV3')
223 IF_EQUAL_RETURN(guid, MFVideoFormat_WVC1); // FCC('WVC1')
224 IF_EQUAL_RETURN(guid, MFVideoFormat_VP90);
225 IF_EQUAL_RETURN(guid, MFVideoFormat_Y210); // FCC('Y210')
226 IF_EQUAL_RETURN(guid, MFVideoFormat_Y216); // FCC('Y216')
227 IF_EQUAL_RETURN(guid, MFVideoFormat_Y410); // FCC('Y410')
228 IF_EQUAL_RETURN(guid, MFVideoFormat_Y416); // FCC('Y416')
229 IF_EQUAL_RETURN(guid, MFVideoFormat_Y41P);
230 IF_EQUAL_RETURN(guid, MFVideoFormat_Y41T);
231 IF_EQUAL_RETURN(guid, MFVideoFormat_YUY2); // FCC('YUY2')
232 IF_EQUAL_RETURN(guid, MFVideoFormat_YV12); // FCC('YV12')
233 IF_EQUAL_RETURN(guid, MFVideoFormat_YVYU);
234
235 IF_EQUAL_RETURN(guid, MFAudioFormat_PCM); // WAVE_FORMAT_PCM
236 IF_EQUAL_RETURN(guid, MFAudioFormat_Float); // WAVE_FORMAT_IEEE_FLOAT
237 IF_EQUAL_RETURN(guid, MFAudioFormat_DTS); // WAVE_FORMAT_DTS
238 IF_EQUAL_RETURN(guid, MFAudioFormat_Dolby_AC3_SPDIF); // WAVE_FORMAT_DOLBY_AC3_SPDIF
239 IF_EQUAL_RETURN(guid, MFAudioFormat_DRM); // WAVE_FORMAT_DRM
240 IF_EQUAL_RETURN(guid, MFAudioFormat_WMAudioV8); // WAVE_FORMAT_WMAUDIO2
241 IF_EQUAL_RETURN(guid, MFAudioFormat_WMAudioV9); // WAVE_FORMAT_WMAUDIO3
242 IF_EQUAL_RETURN(guid, MFAudioFormat_WMAudio_Lossless); // WAVE_FORMAT_WMAUDIO_LOSSLESS
243 IF_EQUAL_RETURN(guid, MFAudioFormat_WMASPDIF); // WAVE_FORMAT_WMASPDIF
244 IF_EQUAL_RETURN(guid, MFAudioFormat_MSP1); // WAVE_FORMAT_WMAVOICE9
245 IF_EQUAL_RETURN(guid, MFAudioFormat_MP3); // WAVE_FORMAT_MPEGLAYER3
246 IF_EQUAL_RETURN(guid, MFAudioFormat_MPEG); // WAVE_FORMAT_MPEG
247 IF_EQUAL_RETURN(guid, MFAudioFormat_AAC); // WAVE_FORMAT_MPEG_HEAAC
248 IF_EQUAL_RETURN(guid, MFAudioFormat_ADTS); // WAVE_FORMAT_MPEG_ADTS_AAC
249
250 return "<unknown>";
251}
252
253static IDataProvider::mfx_codec_id_type convert_to_mfx_codec_id(const GUID& guid) {
254 if (guid == MFVideoFormat_H264) {
255 return MFX_CODEC_AVC;
256 } else if (guid == MFVideoFormat_H265 ||
257 guid == MFVideoFormat_HEVC ||
258 guid == MFVideoFormat_HEVC_ES) {
259 return MFX_CODEC_HEVC;
260 } else if (guid == MFAudioFormat_MPEG) {
261 return MFX_CODEC_MPEG2;
262 } else if (guid == MFVideoFormat_WVC1) {
263 return MFX_CODEC_VC1;
264 } else if (guid == MFVideoFormat_VP90) {
265 return MFX_CODEC_VP9;
266 } else if (guid == MFVideoFormat_AV1) {
267 return MFX_CODEC_AV1;
268 } else if (guid == MFVideoFormat_MJPG) {
269 return MFX_CODEC_JPEG;
270 }
271
272 throw DataProviderUnsupportedException(std::string("unsupported codec type: ") +
273 GetGUIDNameConst(guid));
274}
275
276bool MFPAsyncDemuxDataProvider::select_supported_video_stream(
277 ComPtrGuard<IMFPresentationDescriptor> &descriptor,
278 mfx_codec_id_type &out_codec_id,
279 void *source_id) {
280 DWORD stream_count = 0;
281 BOOL is_stream_selected = false;
282 descriptor->GetStreamDescriptorCount(&stream_count);
283 GAPI_LOG_DEBUG(nullptr, "[" << source_id << "] " <<
284 "received stream count: " << stream_count);
285 for (DWORD stream_index = 0;
286 stream_index < stream_count && !is_stream_selected; stream_index++) {
287
288 GAPI_LOG_DEBUG(nullptr, "[" << source_id << "] " <<
289 "check stream info by index: " << stream_index);
290 IMFStreamDescriptor *stream_descriptor_tmp = nullptr;
291 descriptor->GetStreamDescriptorByIndex(stream_index, &is_stream_selected,
292 &stream_descriptor_tmp);
293 if (!stream_descriptor_tmp) {
294 GAPI_LOG_WARNING(nullptr, "[" << source_id << "] " <<
295 "Cannot get stream descriptor by index: " <<
296 stream_index);
297 continue;
298 }
299
300 ComPtrGuard<IMFStreamDescriptor> stream_descriptor =
301 createCOMPtrGuard(stream_descriptor_tmp);
302 is_stream_selected = false; // deselect until supported stream found
303 IMFMediaTypeHandler *handler_tmp = nullptr;
304 stream_descriptor->GetMediaTypeHandler(&handler_tmp);
305 if (!handler_tmp) {
306 GAPI_LOG_WARNING(nullptr, "[" << source_id << "] " <<
307 "Cannot get media type handler for stream by index: " <<
308 stream_index);
309 continue;
310 }
311
312 ComPtrGuard<IMFMediaTypeHandler> handler = createCOMPtrGuard(handler_tmp);
313 GUID guidMajorType;
314 if (FAILED(handler->GetMajorType(&guidMajorType))) {
315 GAPI_LOG_WARNING(nullptr, "[" << source_id << "] " <<
316 "Cannot get major GUID type for stream by index: " <<
317 stream_index);
318 continue;
319 }
320
321 if (guidMajorType != MFMediaType_Video) {
322 GAPI_LOG_DEBUG(nullptr, "[" << source_id << "] " <<
323 "Skipping non-video stream");
324 continue;
325 }
326 GAPI_LOG_DEBUG(nullptr, "[" << source_id << "] " <<
327 "video stream detected");
328 IMFMediaType *media_type_tmp = nullptr;
329 handler->GetCurrentMediaType(&media_type_tmp);
330 if (!media_type_tmp) {
331 GAPI_LOG_WARNING(nullptr, "[" << source_id << "] " <<
332 "Cannot determine media type for stream by index: " <<
333 stream_index);
334 continue;
335 }
336
337 ComPtrGuard<IMFMediaType> media_type = createCOMPtrGuard(media_type_tmp);
338 GUID subtype;
339 if (SUCCEEDED(media_type->GetGUID(MF_MT_SUBTYPE, &subtype))) {
340 GAPI_LOG_DEBUG(nullptr, "[" << source_id << "] " <<
341 "video type: " << GetGUIDNameConst(subtype));
342
343 std::string is_codec_supported("unsupported, skip...");
344 try {
345 out_codec_id = convert_to_mfx_codec_id(subtype);
346 is_stream_selected = true;
347 is_codec_supported = "selected!";
348 } catch (...) {}
349
350 GAPI_LOG_INFO(nullptr, "[" << source_id << "] " <<
351 "video stream index: " << stream_index <<
352 ", codec: " << GetGUIDNameConst(subtype) <<
353 " - " << is_codec_supported)
354 } else {
355 GAPI_LOG_WARNING(nullptr, "[" << source_id << "] " <<
356 "Cannot get media GUID subtype for stream by index: " <<
357 stream_index);
358 continue;
359 }
360 }
361 return is_stream_selected;
362}
363
364MFPAsyncDemuxDataProvider::MFPAsyncDemuxDataProvider(const std::string& file_path,
365 size_t keep_preprocessed_buf_count_value) :
366 keep_preprocessed_buf_count(keep_preprocessed_buf_count_value),
367 source(createCOMPtrGuard<IMFMediaSource>()),
368 source_reader(createCOMPtrGuard<IMFSourceReader>()),
369 codec(std::numeric_limits<uint32_t>::max()),
370 provider_state(State::InProgress) {
371
372 submit_read_request.clear();
373 com_interface_reference_count = 1; // object itself
374
375 HRESULT hr = S_OK;
376 hr = MFStartup(MF_VERSION);
377 if (FAILED(hr)) {
378 throw DataProviderSystemErrorException(HRESULT_CODE(hr), "Cannot initialize MFStartup");
379 }
380
381 GAPI_LOG_INFO(nullptr, "[" << this << "] " <<
382 " initializing, URI " << file_path);
383 IMFMediaSource *source_tmp = nullptr;
384 hr = create_media_source(file_path, &source_tmp);
385 if (FAILED(hr)) {
386 throw DataProviderSystemErrorException(HRESULT_CODE(hr), "Cannot create IMFMediaSource");
387 }
388 source.reset(source_tmp);
389
390 GAPI_LOG_DEBUG(nullptr, "[" << this << "] " <<
391 " start creating source attributes");
392 IMFAttributes *attrs_tmp = nullptr;
393
394 // NB: create 2 attributes for disable converters & async callback capability
395 const UINT32 relevant_attributes_count = 2;
396 hr = MFCreateAttributes(&attrs_tmp, relevant_attributes_count);
397 if (FAILED(hr)) {
398 throw DataProviderSystemErrorException(HRESULT_CODE(hr), "MFCreateAttributes failed");
399 }
400
401 ComPtrGuard<IMFAttributes> attributes = createCOMPtrGuard(attrs_tmp);
402 hr = attributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, TRUE);
403
404 // set the callback pointer.
405 if (SUCCEEDED(hr))
406 {
407 hr = attributes->SetUnknown(
408 MF_SOURCE_READER_ASYNC_CALLBACK,
409 this
410 );
411 }
412 if (FAILED(hr)) {
413 throw DataProviderSystemErrorException(HRESULT_CODE(hr), "Cannot set MFP async callback ");
414 }
415
416 GAPI_LOG_DEBUG(nullptr, "[" << this << "] " <<
417 "is getting presentation descriptor");
418 IMFPresentationDescriptor* descriptor_tmp = nullptr;
419 hr = source->CreatePresentationDescriptor(&descriptor_tmp);
420 if (FAILED(hr)) {
421 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
422 "CreatePresentationDescriptor failed");
423 }
424 ComPtrGuard<IMFPresentationDescriptor> descriptor = createCOMPtrGuard(descriptor_tmp);
425 if (!MFPAsyncDemuxDataProvider::select_supported_video_stream(descriptor, codec, this)) {
426 // NB: let's pretty notify clients about list of supported codecs to keep
427 // contract in explicit way to avoid continuous troubleshooting
428 const auto &supported_codecs = get_supported_mfx_codec_ids();
429 std::string ss;
430 for (mfxU32 id : supported_codecs) {
431 ss += mfx_codec_id_to_cstr(id);
432 ss += ", ";
433 }
434 if (!ss.empty()) {
435 ss.erase(ss.size() - 2, 2);
436 }
437
438 GAPI_LOG_WARNING(nullptr, "[" << this << "] "
439 "couldn't find video stream with supported params, "
440 "expected codecs: " << ss);
441 throw DataProviderUnsupportedException("couldn't find supported video stream");
442 }
443
444 GAPI_LOG_DEBUG(nullptr, "[" << this << "] " <<
445 "is creating media source");
446 IMFSourceReader *source_reader_tmp = nullptr;
447 hr = MFCreateSourceReaderFromMediaSource(source.get(), attributes.get(),
448 &source_reader_tmp);
449 if (FAILED(hr)) {
450 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
451 "MFCreateSourceReaderFromMediaSource failed");
452 }
453 source_reader = createCOMPtrGuard(source_reader_tmp);
454
455 GAPI_LOG_DEBUG(nullptr, "[" << this << "] " <<
456 "created IMFSourceReader: " << source_reader);
457
458 // Ask for the first sample.
459 hr = request_next(hr, 0, 0);
460 if (FAILED(hr)) {
461 throw DataProviderSystemErrorException(HRESULT_CODE(hr),
462 "ReadSample failed while requesting initial sample");
463 }
464 GAPI_LOG_INFO(nullptr, "[" << this << "] " <<
465 "initialized");
466}
467
468MFPAsyncDemuxDataProvider::~MFPAsyncDemuxDataProvider() {
469 GAPI_LOG_INFO(nullptr, "[" << this << "] " <<
470 "begin deinitializing");
471
472 flush();
473
474 {
475 std::unique_lock<std::mutex> l(buffer_storage_mutex);
476 GAPI_LOG_INFO(nullptr, "Clean up async storage, count: " <<
477 worker_key_to_buffer_mapping_storage.size());
478 for (auto& buffer : worker_key_to_buffer_mapping_storage) {
479 if (buffer.second) {
480 buffer.second->Unlock();
481 }
482 }
483 worker_key_to_buffer_mapping_storage.clear();
484 }
485
486 GAPI_LOG_INFO(nullptr, "Clean data storage, elapsed buffer count: " <<
487 processing_key_to_buffer_mapping_storage.size());
488 for (auto& buffer : processing_key_to_buffer_mapping_storage) {
489 if (buffer.second) {
490 buffer.second->Unlock();
491 }
492 }
493 processing_key_to_buffer_mapping_storage.clear();
494
495 // release COM object before overall MFP shutdown
496 source_reader.reset();
497 source.reset();
498
499 MFShutdown();
500 GAPI_LOG_INFO(nullptr, "[" << this << "] " <<
501 "deinitialized");
502}
503
504
505ULONG MFPAsyncDemuxDataProvider::AddRef() {
506 // align behavior with InterlockedIncrement
507 return com_interface_reference_count.fetch_add(1) + 1;
508}
509
510ULONG MFPAsyncDemuxDataProvider::Release() {
511 auto count = com_interface_reference_count.fetch_sub(1);
512 GAPI_Assert(count != 0 && "Incorrect reference counting for MFPAsyncDemuxDataProvider");
513 count -= 1; // align behavior with InterlockedDecrement
514 return count;
515}
516
517HRESULT MFPAsyncDemuxDataProvider::QueryInterface(REFIID riid, void** ppv)
518{
519 static const QITAB qit[] =
520 {
521 QITABENT(MFPAsyncDemuxDataProvider, IMFSourceReaderCallback),
522 { 0 },
523 };
524 return QISearch(this, qit, riid, ppv);
525}
526
527
528STDMETHODIMP
529MFPAsyncDemuxDataProvider::OnReadSample(HRESULT status, DWORD,
530 DWORD stream_flag, LONGLONG,
531 IMFSample *sample_ptr) {
532 GAPI_LOG_DEBUG(nullptr, "[" << this << "] status: " << std::to_string(HRESULT_CODE(status)) <<
533 ", stream flags: " << stream_flag <<
534 ", sample: " << sample_ptr);
535 HRESULT hr = S_OK;
536 if (FAILED(status)) {
537 hr = status;
538 }
539
540 // check EOF
541 if (stream_flag & MF_SOURCE_READERF_ENDOFSTREAM) {
542 GAPI_LOG_DEBUG(nullptr, "[" << this << "] EOF");
543
544 // close reader
545 provider_state.store(State::Exhausted);
546 buffer_storage_non_empty_cond.notify_all();
547 return hr;
548 }
549
550 submit_read_request.clear();
551
552 // extract stream data
553 size_t worker_buffer_count = 0;
554 if (SUCCEEDED(hr)) {
555 if (sample_ptr) {
556 // Get the video frame buffer from the sample.
557 IMFMediaBuffer *buffer_ptr = nullptr;
558 hr = sample_ptr->ConvertToContiguousBuffer(&buffer_ptr);
559 GAPI_Assert(SUCCEEDED(hr) &&
560 "MFPAsyncDemuxDataProvider::OnReadSample - ConvertToContiguousBuffer failed");
561
562 DWORD max_buffer_size = 0;
563 DWORD curr_size = 0;
564
565 // lock buffer directly into mfx bitstream
566 std::shared_ptr<mfx_bitstream> staging_stream = std::make_shared<mfx_bitstream>();
567 staging_stream->Data = nullptr;
568
569 hr = buffer_ptr->Lock(&staging_stream->Data, &max_buffer_size, &curr_size);
570 GAPI_Assert(SUCCEEDED(hr) &&
571 "MFPAsyncDemuxDataProvider::OnReadSample - Lock failed");
572
573 staging_stream->MaxLength = max_buffer_size;
574 staging_stream->DataLength = curr_size;
575 staging_stream->CodecId = get_mfx_codec_id();
576
577 GAPI_LOG_DEBUG(nullptr, "[" << this << "] bitstream created, data: " <<
578 static_cast<void*>(staging_stream->Data) <<
579 ", MaxLength: " << staging_stream->MaxLength <<
580 ", DataLength: " << staging_stream->DataLength);
581
582 worker_buffer_count = produce_worker_data(staging_stream->Data,
583 createCOMPtrGuard(buffer_ptr),
584 std::move(staging_stream));
585 }
586 } else {
587 GAPI_LOG_WARNING(nullptr, "[" << this << "] callback failed"
588 ", status: " << std::to_string(HRESULT_CODE(status)) <<
589 ", stream flags: " << stream_flag <<
590 ", sample: " << sample_ptr);
591 }
592
593 hr = request_next(hr, stream_flag, worker_buffer_count);
594 return hr;
595}
596
597size_t MFPAsyncDemuxDataProvider::get_locked_buffer_size() const {
598 std::unique_lock<std::mutex> l(buffer_storage_mutex);
599 return worker_locked_buffer_storage.size();
600}
601
602STDMETHODIMP MFPAsyncDemuxDataProvider::OnEvent(DWORD, IMFMediaEvent *) {
603 return S_OK;
604}
605
606STDMETHODIMP MFPAsyncDemuxDataProvider::OnFlush(DWORD) {
607 provider_state.store(State::Exhausted);
608 buffer_storage_non_empty_cond.notify_all();
609 return S_OK;
610}
611
612void MFPAsyncDemuxDataProvider::flush() {
613 if(source_reader) {
614 GAPI_LOG_INFO(nullptr, "[" << this << "] set flush");
615 source_reader->Flush(static_cast<DWORD>(MF_SOURCE_READER_ALL_STREAMS));
616 }
617
618 size_t iterations = 0;
619 const int waiting_ms = 100;
620 const size_t warning_iteration_wait_count = 300; // approx 30 sec
621 while (provider_state.load() != State::Exhausted) {
622 iterations++;
623 if (iterations > warning_iteration_wait_count) {
624 GAPI_LOG_WARNING(nullptr, "[" << this << "] is still waiting for flush finishing, "
625 "iteration: " << iterations);
626 } else {
627 GAPI_LOG_DEBUG(nullptr, "[" << this << "] is waiting for flush finishing, "
628 "iteration: " << iterations);
629 }
630 std::unique_lock<std::mutex> l(buffer_storage_mutex);
631 buffer_storage_non_empty_cond.wait_for(l, std::chrono::milliseconds(waiting_ms));
632 }
633
634 GAPI_LOG_INFO(nullptr, "[" << this << "] has flushed in: " <<
635 iterations * waiting_ms << "ms interval");
636}
637
638HRESULT MFPAsyncDemuxDataProvider::request_next(HRESULT hr,
639 DWORD stream_flag,
640 size_t worker_buffer_count) {
641 GAPI_LOG_DEBUG(nullptr, "[" << this << "] status: " <<
642 std::to_string(HRESULT_CODE(hr)) <<
643 ", stream flags: " << stream_flag <<
644 ", worker buffer count: (" << worker_buffer_count <<
645 "/" << keep_preprocessed_buf_count << ")");
646 // check gap in stream
647 if (stream_flag & MF_SOURCE_READERF_STREAMTICK ) {
648 GAPI_LOG_INFO(nullptr, "[" << this << "] stream gap detected");
649 return hr;
650 }
651
652 if (FAILED(hr)) {
653 GAPI_LOG_WARNING(nullptr, "[" << this << "] callback error "
654 ", status: " << std::to_string(HRESULT_CODE(hr)) <<
655 ", stream flags: " << stream_flag);
656 }
657
658 // put on worker buffers available ready
659 if (worker_buffer_count < keep_preprocessed_buf_count) {
660 // only one consumer might make submit
661 if (!submit_read_request.test_and_set()) {
662 hr = source_reader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
663 0, NULL, NULL, NULL, NULL);
664 GAPI_LOG_DEBUG(nullptr, "[" << this << "] submit read sample, status: " <<
665 std::to_string(HRESULT_CODE(hr)));
666 }
667 }
668 return hr;
669}
670
671void MFPAsyncDemuxDataProvider::consume_worker_data() {
672 // wait callback exchange
673 std::unique_lock<std::mutex> l(buffer_storage_mutex);
674 buffer_storage_non_empty_cond.wait(l, [this] {
675 bool empty = worker_locked_buffer_storage.empty();
676 if (empty) {
677 if (!submit_read_request.test_and_set()) {
678 (void)source_reader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
679 0, NULL, NULL, NULL, NULL);
680 }
681 } else {
682 worker_key_to_buffer_mapping_storage.swap(processing_key_to_buffer_mapping_storage);
683 worker_locked_buffer_storage.swap(processing_locked_buffer_storage);
684 }
685
686 return !empty || provider_state == State::Exhausted;
687 });
688}
689
690size_t MFPAsyncDemuxDataProvider::produce_worker_data(void *key,
691 ComPtrGuard<IMFMediaBuffer> &&buffer,
692 std::shared_ptr<mfx_bitstream> &&staging_stream) {
693 size_t bitstream_count = 0;
694 size_t worker_buffer_count = 0;
695 {
696 std::unique_lock<std::mutex> l(buffer_storage_mutex);
697
698 // remember sample buffer to keep data safe
699 worker_key_to_buffer_mapping_storage.emplace(key, std::move(buffer));
700 worker_buffer_count = worker_key_to_buffer_mapping_storage.size();
701
702 // remember bitstream for consuming
703 worker_locked_buffer_storage.push(std::move(staging_stream));
704 bitstream_count = worker_locked_buffer_storage.size();
705 buffer_storage_non_empty_cond.notify_all();
706 }
707 GAPI_DbgAssert(worker_buffer_count == bitstream_count &&
708 "worker_key_to_buffer_mapping_storage & worker_locked_buffer_storage"
709 " must be the same size" );
710 GAPI_LOG_DEBUG(nullptr, "[" << this << "] created dmux buffer by key: " <<
711 key << ", ready bitstream count: " <<
712 bitstream_count);
713
714 return worker_buffer_count;
715}
716
717/////////////// IDataProvider methods ///////////////
718IDataProvider::mfx_codec_id_type MFPAsyncDemuxDataProvider::get_mfx_codec_id() const {
719 return codec;
720}
721
722bool MFPAsyncDemuxDataProvider::fetch_bitstream_data(std::shared_ptr<mfx_bitstream> &out_bitsream) {
723 if (empty()) {
724 GAPI_LOG_DEBUG(nullptr, "[" << this << "] empty");
725 return false;
726 }
727
728 // utilize consumed bitstream portion allocated at prev step
729 if (out_bitsream) {
730 // make dmux buffer unlock for not empty bitstream
731 GAPI_LOG_DEBUG(nullptr, "[" << this << "] " <<
732 "bitstream before fetch: " <<
733 out_bitsream.get() <<
734 ", DataOffset: " <<
735 out_bitsream->DataOffset <<
736 ", DataLength: " <<
737 out_bitsream->DataLength);
738 if (out_bitsream->DataOffset < out_bitsream->DataLength) {
739 return true;
740 }
741
742 // cleanup
743 auto it = processing_key_to_buffer_mapping_storage.find(out_bitsream->Data);
744 if (it == processing_key_to_buffer_mapping_storage.end()) {
745 GAPI_LOG_WARNING(nullptr, "[" << this << "] " <<
746 "cannot find appropriate dmux buffer by key: " <<
747 static_cast<void*>(out_bitsream->Data));
748 GAPI_Error("invalid bitstream key");
749 }
750 if (it->second) {
751 it->second->Unlock();
752 }
753 processing_key_to_buffer_mapping_storage.erase(it);
754 }
755
756 // consume new bitstream portion
757 if (processing_locked_buffer_storage.empty() &&
758 provider_state.load() == State::InProgress) {
759 // get worker data collected from another thread
760 consume_worker_data();
761 }
762
763 // EOF check: nothing to process at this point
764 if (processing_locked_buffer_storage.empty()) {
765 GAPI_DbgAssert(provider_state == State::Exhausted && "Source reader must be drained");
766 out_bitsream.reset();
767 return false;
768 }
769
770 out_bitsream = processing_locked_buffer_storage.front();
771 processing_locked_buffer_storage.pop();
772
773 GAPI_LOG_DEBUG(nullptr, "[" << this << "] "
774 "bitstream after fetch: " <<
775 out_bitsream.get() <<
776 ", DataOffset: " <<
777 out_bitsream->DataOffset <<
778 ", DataLength: " <<
779 out_bitsream->DataLength);
780 return true;
781}
782
783bool MFPAsyncDemuxDataProvider::empty() const {
784 return (provider_state.load() == State::Exhausted) &&
785 (processing_locked_buffer_storage.size() == 0) &&
786 (get_locked_buffer_size() == 0);
787}
788#else // _WIN32
789
790MFPAsyncDemuxDataProvider::MFPAsyncDemuxDataProvider(const std::string&) {
791 GAPI_Error("Unsupported: Microsoft Media Foundation is not available");
792}
793IDataProvider::mfx_codec_id_type MFPAsyncDemuxDataProvider::get_mfx_codec_id() const {
794 GAPI_Error("Unsupported: Microsoft Media Foundation is not available");
795 return std::numeric_limits<mfx_codec_id_type>::max();
796}
797
798bool MFPAsyncDemuxDataProvider::fetch_bitstream_data(std::shared_ptr<mfx_bitstream> &) {
799 GAPI_Error("Unsupported: Microsoft Media Foundation is not available");
800 return false;
801}
802
803bool MFPAsyncDemuxDataProvider::empty() const {
804 GAPI_Error("Unsupported: Microsoft Media Foundation is not available");
805 return true;
806}
807#endif // _WIN32
808} // namespace onevpl
809} // namespace wip
810} // namespace gapi
811} // namespace cv
812
813#endif // HAVE_ONEVPL
814

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of opencv/modules/gapi/src/streaming/onevpl/demux/async_mfp_demux_data_provider.cpp