gstreamer_gl/
gl_video_frame.rs

Help
1// Take a look at the license at the top of the repository in the LICENSE file.
2
3use std::{fmt::Debug, marker::PhantomData, mem, ptr};
4
5use crate::{ffi, GLMemoryRef};
6use glib::translate::{from_glib, Borrowed, ToGlibPtr};
7use gst_video::{video_frame::IsVideoFrame, VideoFrameExt};
8
9pub enum Readable {}
10pub enum Writable {}
11
12// TODO: implement copy for videoframes. This would need to go through all the individual memories
13//       and copy them. Some GL textures can be copied, others cannot.
14
15pub trait IsGLVideoFrame: IsVideoFrame + Sized {}
16
17mod sealed {
18    pub trait Sealed {}
19    impl<T: super::IsGLVideoFrame> Sealed for T {}
20}
21
22pub trait GLVideoFrameExt: sealed::Sealed + IsGLVideoFrame {
23    #[inline]
24    fn memory(&self, idx: u32) -> Result<&GLMemoryRef, glib::BoolError> {
25        if idx >= self.info().n_planes() {
26            return Err(glib::bool_error!(
27                "Memory index higher than number of memories"
28            ));
29        }
30
31        unsafe {
32            let ptr = self.as_raw().map[idx as usize].memory;
33            if ffi::gst_is_gl_memory(ptr) == glib::ffi::GTRUE {
34                Ok(GLMemoryRef::from_ptr(ptr as _))
35            } else {
36                Err(glib::bool_error!("Memory is not a GLMemory"))
37            }
38        }
39    }
40
41    #[inline]
42    #[doc(alias = "get_texture_id")]
43    fn texture_id(&self, idx: u32) -> Result<u32, glib::BoolError> {
44        Ok(self.memory(idx)?.texture_id())
45    }
46
47    #[inline]
48    #[doc(alias = "get_texture_format")]
49    fn texture_format(&self, idx: u32) -> Result<crate::GLFormat, glib::BoolError> {
50        Ok(self.memory(idx)?.texture_format())
51    }
52
53    #[inline]
54    #[doc(alias = "get_texture_height")]
55    fn texture_height(&self, idx: u32) -> Result<i32, glib::BoolError> {
56        Ok(self.memory(idx)?.texture_height())
57    }
58
59    #[inline]
60    #[doc(alias = "get_texture_target")]
61    fn texture_target(&self, idx: u32) -> Result<crate::GLTextureTarget, glib::BoolError> {
62        Ok(self.memory(idx)?.texture_target())
63    }
64
65    #[inline]
66    #[doc(alias = "get_texture_width")]
67    fn texture_width(&self, idx: u32) -> Result<i32, glib::BoolError> {
68        Ok(self.memory(idx)?.texture_width())
69    }
70}
71
72impl<O: IsGLVideoFrame> GLVideoFrameExt for O {}
73
74pub struct GLVideoFrame<T> {
75    frame: gst_video::ffi::GstVideoFrame,
76    buffer: gst::Buffer,
77    phantom: PhantomData<T>,
78}
79
80unsafe impl<T> Send for GLVideoFrame<T> {}
81unsafe impl<T> Sync for GLVideoFrame<T> {}
82
83impl<T> IsVideoFrame for GLVideoFrame<T> {
84    #[inline]
85    fn as_raw(&self) -> &gst_video::ffi::GstVideoFrame {
86        &self.frame
87    }
88}
89
90impl<T> IsGLVideoFrame for GLVideoFrame<T> {}
91
92impl<T> Debug for GLVideoFrame<T> {
93    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
94        f.debug_struct("GLVideoFrame")
95            .field("flags", &self.flags())
96            .field("id", &self.id())
97            .field("buffer", &self.buffer())
98            .field("info", &self.info())
99            .finish()
100    }
101}
102
103impl<T> GLVideoFrame<T> {
104    #[inline]
105    pub fn into_buffer(self) -> gst::Buffer {
106        unsafe {
107            let mut s = mem::ManuallyDrop::new(self);
108            let buffer = ptr::read(&s.buffer);
109            gst_video::ffi::gst_video_frame_unmap(&mut s.frame);
110            buffer
111        }
112    }
113
114    #[inline]
115    pub unsafe fn from_glib_full(frame: gst_video::ffi::GstVideoFrame) -> Self {
116        let buffer = gst::Buffer::from_glib_none(frame.buffer);
117        Self {
118            frame,
119            buffer,
120            phantom: PhantomData,
121        }
122    }
123
124    #[inline]
125    pub fn into_raw(self) -> gst_video::ffi::GstVideoFrame {
126        unsafe {
127            let mut s = mem::ManuallyDrop::new(self);
128            ptr::drop_in_place(&mut s.buffer);
129            s.frame
130        }
131    }
132
133    #[inline]
134    pub fn as_video_frame_gl_ref(&self) -> GLVideoFrameRef<&gst::BufferRef> {
135        let frame = unsafe { ptr::read(&self.frame) };
136        GLVideoFrameRef {
137            frame,
138            unmap: false,
139            phantom: PhantomData,
140        }
141    }
142}
143
144impl<T> Drop for GLVideoFrame<T> {
145    #[inline]
146    fn drop(&mut self) {
147        unsafe {
148            gst_video::ffi::gst_video_frame_unmap(&mut self.frame);
149        }
150    }
151}
152
153impl GLVideoFrame<Readable> {
154    #[inline]
155    pub fn from_buffer_readable(
156        buffer: gst::Buffer,
157        info: &gst_video::VideoInfo,
158    ) -> Result<Self, gst::Buffer> {
159        skip_assert_initialized!();
160
161        let n_mem = match buffer_n_gl_memory(buffer.as_ref()) {
162            Some(n) => n,
163            None => return Err(buffer),
164        };
165
166        // FIXME: planes are not memories, in multiview use case,
167        // number of memories = planes * views, but the raw memory is
168        // not exposed in videoframe
169        if n_mem != info.n_planes() {
170            return Err(buffer);
171        }
172
173        unsafe {
174            let mut frame = mem::MaybeUninit::uninit();
175            let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
176                frame.as_mut_ptr(),
177                info.to_glib_none().0 as *mut _,
178                buffer.to_glib_none().0,
179                gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
180                    | gst::ffi::GST_MAP_READ
181                    | ffi::GST_MAP_GL as u32,
182            ));
183
184            if !res {
185                Err(buffer)
186            } else {
187                let mut frame = frame.assume_init();
188                // Reset size/stride/offset to 0 as the memory pointers
189                // are the GL texture ID and accessing them would read
190                // random memory.
191                frame.info.size = 0;
192                frame.info.stride.fill(0);
193                frame.info.offset.fill(0);
194                Ok(Self {
195                    frame,
196                    buffer,
197                    phantom: PhantomData,
198                })
199            }
200        }
201    }
202}
203
204impl GLVideoFrame<Writable> {
205    #[inline]
206    pub fn from_buffer_writable(
207        buffer: gst::Buffer,
208        info: &gst_video::VideoInfo,
209    ) -> Result<Self, gst::Buffer> {
210        skip_assert_initialized!();
211
212        let n_mem = match buffer_n_gl_memory(buffer.as_ref()) {
213            Some(n) => n,
214            None => return Err(buffer),
215        };
216
217        // FIXME: planes are not memories, in multiview use case,
218        // number of memories = planes * views, but the raw memory is
219        // not exposed in videoframe
220        if n_mem != info.n_planes() {
221            return Err(buffer);
222        }
223
224        unsafe {
225            let mut frame = mem::MaybeUninit::uninit();
226            let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
227                frame.as_mut_ptr(),
228                info.to_glib_none().0 as *mut _,
229                buffer.to_glib_none().0,
230                gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
231                    | gst::ffi::GST_MAP_READ
232                    | gst::ffi::GST_MAP_WRITE
233                    | ffi::GST_MAP_GL as u32,
234            ));
235
236            if !res {
237                Err(buffer)
238            } else {
239                let mut frame = frame.assume_init();
240                // Reset size/stride/offset to 0 as the memory pointers
241                // are the GL texture ID and accessing them would read
242                // random memory.
243                frame.info.size = 0;
244                frame.info.stride.fill(0);
245                frame.info.offset.fill(0);
246                Ok(Self {
247                    frame,
248                    buffer,
249                    phantom: PhantomData,
250                })
251            }
252        }
253    }
254
255    #[inline]
256    pub fn memory_mut(&self, idx: u32) -> Result<&mut GLMemoryRef, glib::BoolError> {
257        unsafe { Ok(GLMemoryRef::from_mut_ptr(self.memory(idx)?.as_ptr() as _)) }
258    }
259
260    #[inline]
261    pub fn buffer_mut(&mut self) -> &mut gst::BufferRef {
262        unsafe { gst::BufferRef::from_mut_ptr(self.frame.buffer) }
263    }
264}
265
266pub struct GLVideoFrameRef<T> {
267    frame: gst_video::ffi::GstVideoFrame,
268    unmap: bool,
269    phantom: PhantomData<T>,
270}
271
272unsafe impl<T> Send for GLVideoFrameRef<T> {}
273unsafe impl<T> Sync for GLVideoFrameRef<T> {}
274
275impl<T> IsVideoFrame for GLVideoFrameRef<T> {
276    #[inline]
277    fn as_raw(&self) -> &gst_video::ffi::GstVideoFrame {
278        &self.frame
279    }
280}
281
282impl<T> IsGLVideoFrame for GLVideoFrameRef<T> {}
283
284impl<T> Debug for GLVideoFrameRef<T> {
285    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
286        f.debug_struct("GLVideoFrameRef")
287            .field("flags", &self.flags())
288            .field("id", &self.id())
289            .field("buffer", &unsafe {
290                gst::BufferRef::from_ptr(self.frame.buffer)
291            })
292            .field("info", &self.info())
293            .finish()
294    }
295}
296
297impl<'a> GLVideoFrameRef<&'a gst::BufferRef> {
298    #[inline]
299    pub unsafe fn from_glib_borrow(frame: *const gst_video::ffi::GstVideoFrame) -> Borrowed<Self> {
300        debug_assert!(!frame.is_null());
301
302        let frame = ptr::read(frame);
303        Borrowed::new(Self {
304            frame,
305            unmap: false,
306            phantom: PhantomData,
307        })
308    }
309
310    #[inline]
311    pub unsafe fn from_glib_full(frame: gst_video::ffi::GstVideoFrame) -> Self {
312        Self {
313            frame,
314            unmap: true,
315            phantom: PhantomData,
316        }
317    }
318
319    #[inline]
320    pub fn from_buffer_ref_readable<'b>(
321        buffer: &'a gst::BufferRef,
322        info: &'b gst_video::VideoInfo,
323    ) -> Result<GLVideoFrameRef<&'a gst::BufferRef>, glib::error::BoolError> {
324        skip_assert_initialized!();
325
326        let n_mem = match buffer_n_gl_memory(buffer) {
327            Some(n) => n,
328            None => return Err(glib::bool_error!("Memory is not a GstGLMemory")),
329        };
330
331        // FIXME: planes are not memories, in multiview use case,
332        // number of memories = planes * views, but the raw memory is
333        // not exposed in videoframe
334        if n_mem != info.n_planes() {
335            return Err(glib::bool_error!(
336                "Number of planes and memories is not matching"
337            ));
338        }
339
340        unsafe {
341            let mut frame = mem::MaybeUninit::uninit();
342            let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
343                frame.as_mut_ptr(),
344                info.to_glib_none().0 as *mut _,
345                buffer.as_mut_ptr(),
346                gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
347                    | gst::ffi::GST_MAP_READ
348                    | ffi::GST_MAP_GL as u32,
349            ));
350
351            if !res {
352                Err(glib::bool_error!(
353                    "Failed to fill in the values of GstVideoFrame"
354                ))
355            } else {
356                let mut frame = frame.assume_init();
357                // Reset size/stride/offset to 0 as the memory pointers
358                // are the GL texture ID and accessing them would read
359                // random memory.
360                frame.info.size = 0;
361                frame.info.stride.fill(0);
362                frame.info.offset.fill(0);
363                Ok(Self {
364                    frame,
365                    unmap: true,
366                    phantom: PhantomData,
367                })
368            }
369        }
370    }
371}
372
373impl<'a> GLVideoFrameRef<&'a mut gst::BufferRef> {
374    #[inline]
375    pub unsafe fn from_glib_borrow_mut(frame: *mut gst_video::ffi::GstVideoFrame) -> Self {
376        debug_assert!(!frame.is_null());
377
378        let frame = ptr::read(frame);
379        Self {
380            frame,
381            unmap: false,
382            phantom: PhantomData,
383        }
384    }
385
386    #[inline]
387    pub unsafe fn from_glib_full_mut(frame: gst_video::ffi::GstVideoFrame) -> Self {
388        Self {
389            frame,
390            unmap: true,
391            phantom: PhantomData,
392        }
393    }
394
395    #[inline]
396    pub fn from_buffer_ref_writable<'b>(
397        buffer: &'a mut gst::BufferRef,
398        info: &'b gst_video::VideoInfo,
399    ) -> Result<GLVideoFrameRef<&'a mut gst::BufferRef>, glib::error::BoolError> {
400        skip_assert_initialized!();
401
402        let n_mem = match buffer_n_gl_memory(buffer) {
403            Some(n) => n,
404            None => return Err(glib::bool_error!("Memory is not a GstGLMemory")),
405        };
406
407        // FIXME: planes are not memories, in multiview use case,
408        // number of memories = planes * views, but the raw memory is
409        // not exposed in videoframe
410        if n_mem != info.n_planes() {
411            return Err(glib::bool_error!(
412                "Number of planes and memories is not matching"
413            ));
414        }
415
416        unsafe {
417            let mut frame = mem::MaybeUninit::uninit();
418            let res: bool = from_glib(gst_video::ffi::gst_video_frame_map(
419                frame.as_mut_ptr(),
420                info.to_glib_none().0 as *mut _,
421                buffer.as_mut_ptr(),
422                gst_video::ffi::GST_VIDEO_FRAME_MAP_FLAG_NO_REF
423                    | gst::ffi::GST_MAP_READ
424                    | gst::ffi::GST_MAP_WRITE
425                    | ffi::GST_MAP_GL as u32,
426            ));
427
428            if !res {
429                Err(glib::bool_error!(
430                    "Failed to fill in the values of GstVideoFrame"
431                ))
432            } else {
433                let mut frame = frame.assume_init();
434                // Reset size/stride/offset to 0 as the memory pointers
435                // are the GL texture ID and accessing them would read
436                // random memory.
437                frame.info.size = 0;
438                frame.info.stride.fill(0);
439                frame.info.offset.fill(0);
440                Ok(Self {
441                    frame,
442                    unmap: true,
443                    phantom: PhantomData,
444                })
445            }
446        }
447    }
448
449    #[inline]
450    pub fn buffer_mut(&mut self) -> &mut gst::BufferRef {
451        unsafe { gst::BufferRef::from_mut_ptr(self.frame.buffer) }
452    }
453
454    #[inline]
455    pub fn as_mut_ptr(&mut self) -> *mut gst_video::ffi::GstVideoFrame {
456        &mut self.frame
457    }
458
459    #[inline]
460    pub fn memory_mut(&self, idx: u32) -> Result<&mut GLMemoryRef, glib::BoolError> {
461        unsafe { Ok(GLMemoryRef::from_mut_ptr(self.memory(idx)?.as_ptr() as _)) }
462    }
463}
464
465impl<'a> std::ops::Deref for GLVideoFrameRef<&'a mut gst::BufferRef> {
466    type Target = GLVideoFrameRef<&'a gst::BufferRef>;
467
468    #[inline]
469    fn deref(&self) -> &Self::Target {
470        unsafe { &*(self as *const Self as *const Self::Target) }
471    }
472}
473
474impl<T> Drop for GLVideoFrameRef<T> {
475    #[inline]
476    fn drop(&mut self) {
477        unsafe {
478            if self.unmap {
479                gst_video::ffi::gst_video_frame_unmap(&mut self.frame);
480            }
481        }
482    }
483}
484
485fn buffer_n_gl_memory(buffer: &gst::BufferRef) -> Option<u32> {
486    skip_assert_initialized!();
487    unsafe {
488        let buf = buffer.as_mut_ptr();
489        let num = gst::ffi::gst_buffer_n_memory(buf);
490        for i in 0..num - 1 {
491            let mem = gst::ffi::gst_buffer_peek_memory(buf, i);
492            if ffi::gst_is_gl_memory(mem) != glib::ffi::GTRUE {
493                return None;
494            }
495        }
496        Some(num)
497    }
498}
gstreamer_gl/gl_video_frame.rs

gstreamer_gl/
gl_video_frame.rs
