1// Take a look at the license at the top of the repository in the LICENSE file.
2
3use std::{fmt::Debug, marker::PhantomData, mem, ptr};
4
5use crate::{ffi, GLMemoryRef};
6use glib::translate::{from_glib, Borrowed, ToGlibPtr};
7use gst_video::{video_frame::IsVideoFrame, VideoFrameExt};
8
9pub enum Readable {}
10pub enum Writable {}
11
12// TODO: implement copy for videoframes. This would need to go through all the individual memories
13// and copy them. Some GL textures can be copied, others cannot.
14
15pub trait IsGLVideoFrame: IsVideoFrame + Sized {}
16
17mod sealed {
18 pub trait Sealed {}
19 impl<T: super::IsGLVideoFrame> Sealed for T {}
20}
21
22pub trait GLVideoFrameExt: sealed::Sealed + IsGLVideoFrame {
23 #[inline]
24 fn memory(&self, idx: u32) -> Result<&GLMemoryRef, glib::BoolError> {
25 if idx >= self.info().n_planes() {
26 return Err(glib::bool_error!(
27 "Memory index higher than number of memories"
28 ));
29 }
30
31 unsafe {
32 let ptr = self.as_raw().map[idx as usize].memory;
33 if ffi::gst_is_gl_memory(ptr) == glib::ffi::GTRUE {
34 Ok(GLMemoryRef::from_ptr(ptr as _))
35 } else {
36 Err(glib::bool_error!("Memory is not a GLMemory"))
37 }
38 }
39 }
40
41 #[inline]
42 #[doc(alias = "get_texture_id")]
43 fn texture_id(&self, idx: u32) -> Result<u32, glib::BoolError> {
44 Ok(self.memory(idx)?.texture_id())
45 }
46
47 #[inline]
48 #[doc(alias = "get_texture_format")]
49 fn texture_format(&self, idx: u32) -> Result<crate::GLFormat, glib::BoolError> {
50 Ok(self.memory(idx)?.texture_format())
51 }
52
53 #[inline]
54 #[doc(alias = "get_texture_height")]
55 fn texture_height(&self, idx: u32) -> Result<i32, glib::BoolError> {
56 Ok(self.memory(idx)?.texture_height())
57 }
58
59 #[inline]
60 #[doc(alias = "get_texture_target")]
61 fn texture_target(&self, idx: u32) -> Result<crate::GLTextureTarget, glib::BoolError> {
62 Ok(self.memory(idx)?.texture_target())
63 }
64
65 #[inline]
66 #[doc(alias = "get_texture_width")]
67 fn texture_width(&self, idx: u32) -> Result<i32, glib::BoolError> {
68 Ok(self.memory(idx)?.texture_width())
69 }
70}
71
72impl<O: IsGLVideoFrame> GLVideoFrameExt for O {}
73
74pub struct GLVideoFrame<T> {
75 frame: gst_video::ffi::GstVideoFrame,
76 buffer: gst::Buffer,
77 phantom: PhantomData<T>,
78}
79
80unsafe impl<T> Send for GLVideoFrame<T> {}
81unsafe impl<T> Sync for GLVideoFrame<T> {}
82
83impl<T> IsVideoFrame for GLVideoFrame<T> {
84 #[inline]
85 fn as_raw(&self) -> &gst_video::ffi::GstVideoFrame {
86 &self.frame
87 }
88}
89
90impl<T> IsGLVideoFrame for GLVideoFrame<T> {}
91
92impl<T> Debug for GLVideoFrame<T> {
93 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
94 f&mut DebugStruct<'_, '_>.debug_struct("GLVideoFrame")
95 .field("flags", &self.flags())
96 .field("id", &self.id())
97 .field("buffer", &self.buffer())
98 .field(name:"info", &self.info())
99 .finish()
100 }
101}
102
103impl<T> GLVideoFrame<T> {
104 #[inline]
105 pub fn into_buffer(self) -> gst::Buffer {
106 unsafe {
107 let mut s = mem::ManuallyDrop::new(self);
108 let buffer = ptr::read(&s.buffer);
109 gst_video::ffi::gst_video_frame_unmap(&mut s.frame);
110 buffer
111 }
112 }
113
114 #[inline]
115 pub unsafe fn from_glib_full(frame: gst_video::ffi::GstVideoFrame) -> Self {
116 let buffer = gst::Buffer::from_glib_none(frame.buffer);
117 Self {
118 frame,
119 buffer,
120 phantom: PhantomData,
121 }
122 }
123
124 #[inline]
125 pub fn into_raw(self) -> gst_video::ffi::GstVideoFrame {
126 unsafe {
127 let mut s = mem::ManuallyDrop::new(self);
128 ptr::drop_in_place(&mut s.buffer);
129 s.frame
130 }
131 }
132
133 #[inline]
134 pub fn as_video_frame_gl_ref(&self) -> GLVideoFrameRef<&gst::BufferRef> {
135 let frame = unsafe { ptr::read(&self.frame) };
136 GLVideoFrameRef {
137 frame,
138 unmap: false,
139 phantom: PhantomData,
140 }
141 }
142}
143
144impl<T> Drop for GLVideoFrame<T> {
145 #[inline]
146 fn drop(&mut self) {
147 unsafe {
148 gst_video::ffi::gst_video_frame_unmap(&mut self.frame);
149 }
150 }
151}
152
153impl GLVideoFrame<Readable> {
154 #[inline]
155 pub fn from_buffer_readable(
156 buffer: gst::Buffer,
157 info: &gst_video::VideoInfo,
158 ) -> Result<Self, gst::Buffer> {
159 skip_assert_initialized!();
160
161 let n_mem = match buffer_n_gl_memory(buffer.as_ref()) {
162 Some(n) => n,
163 None => return Err(buffer),
164 };
165
166 // FIXME: planes are not memories, in multiview use case,
167 // number of memories = planes * views, but the raw memory is
168 // not exposed in videoframe
169 if n_mem != info.n_planes() {
170 return Err(buffer);
171 }
172
173 unsafe {
174 let mut frame = mem::MaybeUninit::uninit();
175 let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
176 frame.as_mut_ptr(),
177 info.to_glib_none().0 as *mut _,
178 buffer.to_glib_none().0,
179 gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
180 | gst::ffi::GST_MAP_READ
181 | ffi::GST_MAP_GL as u32,
182 ));
183
184 if !res {
185 Err(buffer)
186 } else {
187 let mut frame = frame.assume_init();
188 // Reset size/stride/offset to 0 as the memory pointers
189 // are the GL texture ID and accessing them would read
190 // random memory.
191 frame.info.size = 0;
192 frame.info.stride.fill(0);
193 frame.info.offset.fill(0);
194 Ok(Self {
195 frame,
196 buffer,
197 phantom: PhantomData,
198 })
199 }
200 }
201 }
202}
203
204impl GLVideoFrame<Writable> {
205 #[inline]
206 pub fn from_buffer_writable(
207 buffer: gst::Buffer,
208 info: &gst_video::VideoInfo,
209 ) -> Result<Self, gst::Buffer> {
210 skip_assert_initialized!();
211
212 let n_mem = match buffer_n_gl_memory(buffer.as_ref()) {
213 Some(n) => n,
214 None => return Err(buffer),
215 };
216
217 // FIXME: planes are not memories, in multiview use case,
218 // number of memories = planes * views, but the raw memory is
219 // not exposed in videoframe
220 if n_mem != info.n_planes() {
221 return Err(buffer);
222 }
223
224 unsafe {
225 let mut frame = mem::MaybeUninit::uninit();
226 let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
227 frame.as_mut_ptr(),
228 info.to_glib_none().0 as *mut _,
229 buffer.to_glib_none().0,
230 gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
231 | gst::ffi::GST_MAP_READ
232 | gst::ffi::GST_MAP_WRITE
233 | ffi::GST_MAP_GL as u32,
234 ));
235
236 if !res {
237 Err(buffer)
238 } else {
239 let mut frame = frame.assume_init();
240 // Reset size/stride/offset to 0 as the memory pointers
241 // are the GL texture ID and accessing them would read
242 // random memory.
243 frame.info.size = 0;
244 frame.info.stride.fill(0);
245 frame.info.offset.fill(0);
246 Ok(Self {
247 frame,
248 buffer,
249 phantom: PhantomData,
250 })
251 }
252 }
253 }
254
255 #[inline]
256 pub fn memory_mut(&self, idx: u32) -> Result<&mut GLMemoryRef, glib::BoolError> {
257 unsafe { Ok(GLMemoryRef::from_mut_ptr(self.memory(idx)?.as_ptr() as _)) }
258 }
259
260 #[inline]
261 pub fn buffer_mut(&mut self) -> &mut gst::BufferRef {
262 unsafe { gst::BufferRef::from_mut_ptr(self.frame.buffer) }
263 }
264}
265
266pub struct GLVideoFrameRef<T> {
267 frame: gst_video::ffi::GstVideoFrame,
268 unmap: bool,
269 phantom: PhantomData<T>,
270}
271
272unsafe impl<T> Send for GLVideoFrameRef<T> {}
273unsafe impl<T> Sync for GLVideoFrameRef<T> {}
274
275impl<T> IsVideoFrame for GLVideoFrameRef<T> {
276 #[inline]
277 fn as_raw(&self) -> &gst_video::ffi::GstVideoFrame {
278 &self.frame
279 }
280}
281
282impl<T> IsGLVideoFrame for GLVideoFrameRef<T> {}
283
284impl<T> Debug for GLVideoFrameRef<T> {
285 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
286 f&mut DebugStruct<'_, '_>.debug_struct("GLVideoFrameRef")
287 .field("flags", &self.flags())
288 .field("id", &self.id())
289 .field("buffer", &unsafe {
290 gst::BufferRef::from_ptr(self.frame.buffer)
291 })
292 .field(name:"info", &self.info())
293 .finish()
294 }
295}
296
297impl<'a> GLVideoFrameRef<&'a gst::BufferRef> {
298 #[inline]
299 pub unsafe fn from_glib_borrow(frame: *const gst_video::ffi::GstVideoFrame) -> Borrowed<Self> {
300 debug_assert!(!frame.is_null());
301
302 let frame = ptr::read(frame);
303 Borrowed::new(Self {
304 frame,
305 unmap: false,
306 phantom: PhantomData,
307 })
308 }
309
310 #[inline]
311 pub unsafe fn from_glib_full(frame: gst_video::ffi::GstVideoFrame) -> Self {
312 Self {
313 frame,
314 unmap: true,
315 phantom: PhantomData,
316 }
317 }
318
319 #[inline]
320 pub fn from_buffer_ref_readable<'b>(
321 buffer: &'a gst::BufferRef,
322 info: &'b gst_video::VideoInfo,
323 ) -> Result<GLVideoFrameRef<&'a gst::BufferRef>, glib::error::BoolError> {
324 skip_assert_initialized!();
325
326 let n_mem = match buffer_n_gl_memory(buffer) {
327 Some(n) => n,
328 None => return Err(glib::bool_error!("Memory is not a GstGLMemory")),
329 };
330
331 // FIXME: planes are not memories, in multiview use case,
332 // number of memories = planes * views, but the raw memory is
333 // not exposed in videoframe
334 if n_mem != info.n_planes() {
335 return Err(glib::bool_error!(
336 "Number of planes and memories is not matching"
337 ));
338 }
339
340 unsafe {
341 let mut frame = mem::MaybeUninit::uninit();
342 let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
343 frame.as_mut_ptr(),
344 info.to_glib_none().0 as *mut _,
345 buffer.as_mut_ptr(),
346 gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
347 | gst::ffi::GST_MAP_READ
348 | ffi::GST_MAP_GL as u32,
349 ));
350
351 if !res {
352 Err(glib::bool_error!(
353 "Failed to fill in the values of GstVideoFrame"
354 ))
355 } else {
356 let mut frame = frame.assume_init();
357 // Reset size/stride/offset to 0 as the memory pointers
358 // are the GL texture ID and accessing them would read
359 // random memory.
360 frame.info.size = 0;
361 frame.info.stride.fill(0);
362 frame.info.offset.fill(0);
363 Ok(Self {
364 frame,
365 unmap: true,
366 phantom: PhantomData,
367 })
368 }
369 }
370 }
371}
372
373impl<'a> GLVideoFrameRef<&'a mut gst::BufferRef> {
374 #[inline]
375 pub unsafe fn from_glib_borrow_mut(frame: *mut gst_video::ffi::GstVideoFrame) -> Self {
376 debug_assert!(!frame.is_null());
377
378 let frame = ptr::read(frame);
379 Self {
380 frame,
381 unmap: false,
382 phantom: PhantomData,
383 }
384 }
385
386 #[inline]
387 pub unsafe fn from_glib_full_mut(frame: gst_video::ffi::GstVideoFrame) -> Self {
388 Self {
389 frame,
390 unmap: true,
391 phantom: PhantomData,
392 }
393 }
394
395 #[inline]
396 pub fn from_buffer_ref_writable<'b>(
397 buffer: &'a mut gst::BufferRef,
398 info: &'b gst_video::VideoInfo,
399 ) -> Result<GLVideoFrameRef<&'a mut gst::BufferRef>, glib::error::BoolError> {
400 skip_assert_initialized!();
401
402 let n_mem = match buffer_n_gl_memory(buffer) {
403 Some(n) => n,
404 None => return Err(glib::bool_error!("Memory is not a GstGLMemory")),
405 };
406
407 // FIXME: planes are not memories, in multiview use case,
408 // number of memories = planes * views, but the raw memory is
409 // not exposed in videoframe
410 if n_mem != info.n_planes() {
411 return Err(glib::bool_error!(
412 "Number of planes and memories is not matching"
413 ));
414 }
415
416 unsafe {
417 let mut frame = mem::MaybeUninit::uninit();
418 let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
419 frame.as_mut_ptr(),
420 info.to_glib_none().0 as *mut _,
421 buffer.as_mut_ptr(),
422 gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
423 | gst::ffi::GST_MAP_READ
424 | gst::ffi::GST_MAP_WRITE
425 | ffi::GST_MAP_GL as u32,
426 ));
427
428 if !res {
429 Err(glib::bool_error!(
430 "Failed to fill in the values of GstVideoFrame"
431 ))
432 } else {
433 let mut frame = frame.assume_init();
434 // Reset size/stride/offset to 0 as the memory pointers
435 // are the GL texture ID and accessing them would read
436 // random memory.
437 frame.info.size = 0;
438 frame.info.stride.fill(0);
439 frame.info.offset.fill(0);
440 Ok(Self {
441 frame,
442 unmap: true,
443 phantom: PhantomData,
444 })
445 }
446 }
447 }
448
449 #[inline]
450 pub fn buffer_mut(&mut self) -> &mut gst::BufferRef {
451 unsafe { gst::BufferRef::from_mut_ptr(self.frame.buffer) }
452 }
453
454 #[inline]
455 pub fn as_mut_ptr(&mut self) -> *mut gst_video::ffi::GstVideoFrame {
456 &mut self.frame
457 }
458
459 #[inline]
460 pub fn memory_mut(&self, idx: u32) -> Result<&mut GLMemoryRef, glib::BoolError> {
461 unsafe { Ok(GLMemoryRef::from_mut_ptr(self.memory(idx)?.as_ptr() as _)) }
462 }
463}
464
465impl<'a> std::ops::Deref for GLVideoFrameRef<&'a mut gst::BufferRef> {
466 type Target = GLVideoFrameRef<&'a gst::BufferRef>;
467
468 #[inline]
469 fn deref(&self) -> &Self::Target {
470 unsafe { &*(self as *const Self as *const Self::Target) }
471 }
472}
473
474impl<T> Drop for GLVideoFrameRef<T> {
475 #[inline]
476 fn drop(&mut self) {
477 unsafe {
478 if self.unmap {
479 gst_video::ffi::gst_video_frame_unmap(&mut self.frame);
480 }
481 }
482 }
483}
484
485fn buffer_n_gl_memory(buffer: &gst::BufferRef) -> Option<u32> {
486 skip_assert_initialized!();
487 unsafe {
488 let buf: *mut GstBuffer = buffer.as_mut_ptr();
489 let num: u32 = gst::ffi::gst_buffer_n_memory(buffer:buf);
490 for i: u32 in 0..num - 1 {
491 let mem: *mut GstMemory = gst::ffi::gst_buffer_peek_memory(buffer:buf, idx:i);
492 if ffi::gst_is_gl_memory(mem) != glib::ffi::GTRUE {
493 return None;
494 }
495 }
496 Some(num)
497 }
498}
499