From 8d99b44a05b707605c9afc0d11dca4f3b5fe6859 Mon Sep 17 00:00:00 2001 From: Slawomir Rosek Date: Tue, 28 May 2024 14:37:12 +0200 Subject: [PATCH 1/7] codec/h264: Add missing V4l2 fields to SliceHeader This commit provides some extra V4L2 fields to the SliceHeader which are calculated by parser while processing h264 stream. --- src/codec/h264/parser.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/codec/h264/parser.rs b/src/codec/h264/parser.rs index b486aaf0..1eb74609 100644 --- a/src/codec/h264/parser.rs +++ b/src/codec/h264/parser.rs @@ -266,6 +266,10 @@ pub struct SliceHeader { /// the bottom field of a coded frame specified in clause 8.2.1. pub delta_pic_order_cnt: [i32; 2], + /// This value is required by V4L2 stateless decode params so it is calculated + /// by parser while processing slice header. + pub pic_order_cnt_bit_size: usize, + /// Shall be equal to 0 for slices and slice data partitions belonging to /// the primary coded picture. The value of `redundant_pic_cnt shall` be /// greater than 0 for coded slices or coded slice data partitions of a @@ -315,6 +319,10 @@ pub struct SliceHeader { /// Decoded reference picture marking parsed using 7.3.3.3 pub dec_ref_pic_marking: RefPicMarking, + /// This value is required by V4L2 stateless decode params so it is calculated + /// by parser while processing slice header. + pub dec_ref_pic_marking_bit_size: usize, + /// Specifies the index for determining the initialization table used in the /// initialization process for context variables. pub cabac_init_idc: u8, @@ -2334,6 +2342,7 @@ impl Parser { ) -> anyhow::Result<()> { let rpm = &mut header.dec_ref_pic_marking; + let num_bits_left = r.num_bits_left(); if nalu.header.idr_pic_flag { rpm.no_output_of_prior_pics_flag = r.read_bit()?; rpm.long_term_reference_flag = r.read_bit()?; @@ -2372,6 +2381,7 @@ impl Parser { } } } + header.dec_ref_pic_marking_bit_size = num_bits_left - r.num_bits_left(); Ok(()) } @@ -2436,6 +2446,7 @@ impl Parser { header.idr_pic_id = r.read_ue_max(0xffff)?; } + let num_bits_left = r.num_bits_left(); if sps.pic_order_cnt_type == 0 { header.pic_order_cnt_lsb = r.read_bits(usize::from(sps.log2_max_pic_order_cnt_lsb_minus4) + 4)?; @@ -2451,6 +2462,7 @@ impl Parser { header.delta_pic_order_cnt[1] = r.read_se()?; } } + header.pic_order_cnt_bit_size = num_bits_left - r.num_bits_left(); if pps.redundant_pic_cnt_present_flag { header.redundant_pic_cnt = r.read_ue_max(127)?; From 7ca7c7db3d99aba94c2d5ad28e7c558a560acafb Mon Sep 17 00:00:00 2001 From: Slawomir Rosek Date: Thu, 6 Jun 2024 10:27:03 +0200 Subject: [PATCH 2/7] device/v4l2: Add v4l2 stateless device abstraction This commit provides initial implementation of v4l2 stateless device implemented on top of the v4l2r library. This abstraction layer is intended to be used directly by v4l2 backend implementation. TODO: * handle video device path (it's temporary set to /dev/video-dec0) * probe media device path (it's temporary set to /dev/video-med0) * handle memory backends other than mmap * handle video formats other than h264 * handle queue start/stop at runtime * handle DRC at runtime --- src/device.rs | 6 + src/device/v4l2.rs | 5 + src/device/v4l2/stateless.rs | 8 + src/device/v4l2/stateless/controls.rs | 5 + src/device/v4l2/stateless/controls/h264.rs | 326 +++++++++++++++++++++ src/device/v4l2/stateless/device.rs | 147 ++++++++++ src/device/v4l2/stateless/queue.rs | 321 ++++++++++++++++++++ src/device/v4l2/stateless/request.rs | 206 +++++++++++++ src/lib.rs | 1 + 9 files changed, 1025 insertions(+) create mode 100644 src/device.rs create mode 100644 src/device/v4l2.rs create mode 100644 src/device/v4l2/stateless.rs create mode 100644 src/device/v4l2/stateless/controls.rs create mode 100644 src/device/v4l2/stateless/controls/h264.rs create mode 100644 src/device/v4l2/stateless/device.rs create mode 100644 src/device/v4l2/stateless/queue.rs create mode 100644 src/device/v4l2/stateless/request.rs diff --git a/src/device.rs b/src/device.rs new file mode 100644 index 00000000..e82a3308 --- /dev/null +++ b/src/device.rs @@ -0,0 +1,6 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#[cfg(feature = "v4l2")] +pub mod v4l2; diff --git a/src/device/v4l2.rs b/src/device/v4l2.rs new file mode 100644 index 00000000..9ce03c2e --- /dev/null +++ b/src/device/v4l2.rs @@ -0,0 +1,5 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +pub mod stateless; diff --git a/src/device/v4l2/stateless.rs b/src/device/v4l2/stateless.rs new file mode 100644 index 00000000..0a6a11b2 --- /dev/null +++ b/src/device/v4l2/stateless.rs @@ -0,0 +1,8 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +pub mod controls; +pub mod device; +pub mod queue; +pub mod request; diff --git a/src/device/v4l2/stateless/controls.rs b/src/device/v4l2/stateless/controls.rs new file mode 100644 index 00000000..002af849 --- /dev/null +++ b/src/device/v4l2/stateless/controls.rs @@ -0,0 +1,5 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +pub mod h264; diff --git a/src/device/v4l2/stateless/controls/h264.rs b/src/device/v4l2/stateless/controls/h264.rs new file mode 100644 index 00000000..0aca98c0 --- /dev/null +++ b/src/device/v4l2/stateless/controls/h264.rs @@ -0,0 +1,326 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use crate::codec::h264::parser::Pps; +use crate::codec::h264::parser::SliceHeader; +use crate::codec::h264::parser::Sps; +use crate::codec::h264::picture::Field; +use crate::codec::h264::picture::IsIdr; +use crate::codec::h264::picture::PictureData; +use crate::codec::h264::picture::RcPictureData; +use crate::codec::h264::picture::Reference; + +use v4l2r::bindings::v4l2_ctrl_h264_decode_params; +use v4l2r::bindings::v4l2_ctrl_h264_pps; +use v4l2r::bindings::v4l2_ctrl_h264_scaling_matrix; +use v4l2r::bindings::v4l2_ctrl_h264_sps; +use v4l2r::bindings::v4l2_h264_dpb_entry; +use v4l2r::bindings::v4l2_stateless_h264_decode_mode_V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED as V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED; +use v4l2r::bindings::v4l2_stateless_h264_decode_mode_V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED as V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED; +use v4l2r::bindings::V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD; +use v4l2r::bindings::V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC; +use v4l2r::bindings::V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; +use v4l2r::bindings::V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; +use v4l2r::bindings::V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; +use v4l2r::bindings::V4L2_H264_DPB_ENTRY_FLAG_VALID; +use v4l2r::bindings::V4L2_H264_FRAME_REF; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; +use v4l2r::bindings::V4L2_H264_PPS_FLAG_WEIGHTED_PRED; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET0_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET1_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET2_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET3_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET4_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_CONSTRAINT_SET5_FLAG; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; +use v4l2r::bindings::V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; +use v4l2r::controls::codec::H264DecodeMode; +use v4l2r::controls::codec::H264DecodeParams; +use v4l2r::controls::codec::H264Pps; +use v4l2r::controls::codec::H264ScalingMatrix; +use v4l2r::controls::codec::H264Sps; +use v4l2r::controls::SafeExtControl; + +impl From<&Sps> for v4l2_ctrl_h264_sps { + fn from(sps: &Sps) -> Self { + let mut constraint_set_flags: u32 = 0; + if sps.constraint_set0_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET0_FLAG; + } + if sps.constraint_set1_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET1_FLAG; + } + if sps.constraint_set2_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET2_FLAG; + } + if sps.constraint_set3_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET3_FLAG; + } + if sps.constraint_set4_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET4_FLAG; + } + if sps.constraint_set5_flag { + constraint_set_flags |= V4L2_H264_SPS_CONSTRAINT_SET5_FLAG; + } + let mut flags: u32 = 0; + if sps.separate_colour_plane_flag { + flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; + } + if sps.qpprime_y_zero_transform_bypass_flag { + flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; + } + if sps.delta_pic_order_always_zero_flag { + flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; + } + if sps.gaps_in_frame_num_value_allowed_flag { + flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; + } + if sps.frame_mbs_only_flag { + flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; + } + if sps.mb_adaptive_frame_field_flag { + flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; + } + if sps.direct_8x8_inference_flag { + flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; + } + Self { + profile_idc: sps.profile_idc, + constraint_set_flags: constraint_set_flags as u8, + level_idc: sps.level_idc as u8, + seq_parameter_set_id: sps.seq_parameter_set_id, + chroma_format_idc: sps.chroma_format_idc, + bit_depth_luma_minus8: sps.bit_depth_luma_minus8, + bit_depth_chroma_minus8: sps.bit_depth_chroma_minus8, + log2_max_frame_num_minus4: sps.log2_max_frame_num_minus4, + pic_order_cnt_type: sps.pic_order_cnt_type, + log2_max_pic_order_cnt_lsb_minus4: sps.log2_max_pic_order_cnt_lsb_minus4, + max_num_ref_frames: sps.max_num_ref_frames as u8, + num_ref_frames_in_pic_order_cnt_cycle: sps.num_ref_frames_in_pic_order_cnt_cycle, + offset_for_ref_frame: sps.offset_for_ref_frame, + offset_for_non_ref_pic: sps.offset_for_non_ref_pic, + offset_for_top_to_bottom_field: sps.offset_for_top_to_bottom_field, + pic_width_in_mbs_minus1: sps.pic_width_in_mbs_minus1 as u16, + pic_height_in_map_units_minus1: sps.pic_height_in_map_units_minus1 as u16, + flags, + ..Default::default() + } + } +} + +impl From<&Pps> for v4l2_ctrl_h264_pps { + fn from(pps: &Pps) -> Self { + let mut flags: u32 = 0; + if pps.entropy_coding_mode_flag { + flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; + } + if pps.bottom_field_pic_order_in_frame_present_flag { + flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; + } + if pps.weighted_pred_flag { + flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; + } + if pps.deblocking_filter_control_present_flag { + flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; + } + if pps.constrained_intra_pred_flag { + flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; + } + if pps.redundant_pic_cnt_present_flag { + flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; + } + if pps.transform_8x8_mode_flag { + flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; + } + if pps.pic_scaling_matrix_present_flag { + flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; + } + Self { + pic_parameter_set_id: pps.pic_parameter_set_id, + seq_parameter_set_id: pps.seq_parameter_set_id, + num_slice_groups_minus1: pps.num_slice_groups_minus1 as u8, + num_ref_idx_l0_default_active_minus1: pps.num_ref_idx_l0_default_active_minus1, + num_ref_idx_l1_default_active_minus1: pps.num_ref_idx_l1_default_active_minus1, + weighted_bipred_idc: pps.weighted_bipred_idc, + pic_init_qp_minus26: pps.pic_init_qp_minus26, + pic_init_qs_minus26: pps.pic_init_qs_minus26, + chroma_qp_index_offset: pps.chroma_qp_index_offset, + second_chroma_qp_index_offset: pps.second_chroma_qp_index_offset, + flags: flags as u16, + ..Default::default() + } + } +} + +pub struct V4l2CtrlH264DpbEntry { + pub timestamp: u64, + pub pic: RcPictureData, +} + +impl From<&V4l2CtrlH264DpbEntry> for v4l2_h264_dpb_entry { + fn from(dpb: &V4l2CtrlH264DpbEntry) -> Self { + let pic: &PictureData = &dpb.pic.borrow(); + // TODO DCHECK_EQ(pic->field, H264Picture::FIELD_NONE) + // TODO << "Interlacing not supported"; + + let (frame_num, pic_num): (u16, u32) = match pic.reference() { + Reference::LongTerm => (pic.long_term_pic_num as u16, pic.long_term_frame_idx), + _ => (pic.frame_num as u16, pic.pic_num as u32), + }; + + let mut flags: u32 = V4L2_H264_DPB_ENTRY_FLAG_VALID; + if pic.nal_ref_idc != 0 { + flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; + } + if matches!(pic.reference(), Reference::LongTerm) { + flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; + } + + Self { + reference_ts: dpb.timestamp * 1000, // usec to nsec + frame_num, + pic_num, + fields: V4L2_H264_FRAME_REF as u8, + top_field_order_cnt: pic.top_field_order_cnt, + bottom_field_order_cnt: pic.bottom_field_order_cnt, + flags, + ..Default::default() + } + } +} + +#[derive(Default)] +pub struct V4l2CtrlH264Sps { + handle: v4l2_ctrl_h264_sps, +} + +impl V4l2CtrlH264Sps { + pub fn new() -> Self { + Default::default() + } + pub fn set(&mut self, sps: &Sps) -> &mut Self { + self.handle = v4l2_ctrl_h264_sps::from(sps); + self + } +} + +impl From<&V4l2CtrlH264Sps> for SafeExtControl { + fn from(sps: &V4l2CtrlH264Sps) -> Self { + SafeExtControl::::from(sps.handle) + } +} + +#[derive(Default)] +pub struct V4l2CtrlH264Pps { + handle: v4l2_ctrl_h264_pps, +} + +impl V4l2CtrlH264Pps { + pub fn new() -> Self { + Default::default() + } + pub fn set(&mut self, pps: &Pps) -> &mut Self { + self.handle = v4l2_ctrl_h264_pps::from(pps); + self + } +} + +impl From<&V4l2CtrlH264Pps> for SafeExtControl { + fn from(pps: &V4l2CtrlH264Pps) -> Self { + SafeExtControl::::from(pps.handle) + } +} + +#[derive(Default)] +pub struct V4l2CtrlH264ScalingMatrix { + handle: v4l2_ctrl_h264_scaling_matrix, +} + +impl V4l2CtrlH264ScalingMatrix { + pub fn new() -> Self { + Default::default() + } + pub fn set(&mut self) -> &mut Self { + todo!() + } +} + +impl From<&V4l2CtrlH264ScalingMatrix> for SafeExtControl { + fn from(scaling_matrix: &V4l2CtrlH264ScalingMatrix) -> Self { + SafeExtControl::::from(scaling_matrix.handle) + } +} + +#[derive(Default)] +pub struct V4l2CtrlH264DecodeParams { + handle: v4l2_ctrl_h264_decode_params, +} + +impl V4l2CtrlH264DecodeParams { + pub fn new() -> Self { + Default::default() + } + pub fn set_picture_data(&mut self, pic: &PictureData) -> &mut Self { + self.handle.top_field_order_cnt = pic.top_field_order_cnt; + self.handle.bottom_field_order_cnt = pic.bottom_field_order_cnt; + self.handle.flags |= match pic.field { + Field::Top => V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC, + Field::Bottom => { + V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC | V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD + } + _ => 0, + }; + self.handle.flags |= match pic.is_idr { + IsIdr::Yes { idr_pic_id: _ } => V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC, + _ => 0, + }; + self.handle.nal_ref_idc = pic.nal_ref_idc as u16; + self + } + pub fn set_dpb_entries(&mut self, dpb: Vec) -> &mut Self { + for i in 0..dpb.len() { + self.handle.dpb[i] = v4l2_h264_dpb_entry::from(&dpb[i]); + } + self + } + pub fn set_slice_header(&mut self, slice_header: &SliceHeader) -> &mut Self { + self.handle.frame_num = slice_header.frame_num; + self.handle.idr_pic_id = slice_header.idr_pic_id; + self.handle.pic_order_cnt_lsb = slice_header.pic_order_cnt_lsb; + self.handle.delta_pic_order_cnt_bottom = slice_header.delta_pic_order_cnt_bottom; + self.handle.delta_pic_order_cnt0 = slice_header.delta_pic_order_cnt[0]; + self.handle.delta_pic_order_cnt1 = slice_header.delta_pic_order_cnt[1]; + self.handle.dec_ref_pic_marking_bit_size = slice_header.dec_ref_pic_marking_bit_size as u32; + self.handle.pic_order_cnt_bit_size = slice_header.pic_order_cnt_bit_size as u32; + self + } +} + +impl From<&V4l2CtrlH264DecodeParams> for SafeExtControl { + fn from(decode_params: &V4l2CtrlH264DecodeParams) -> Self { + SafeExtControl::::from(decode_params.handle) + } +} + +pub enum V4l2CtrlH264DecodeMode { + SliceBased = V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED as isize, + FrameBased = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED as isize, +} + +impl From for SafeExtControl { + fn from(decode_mode: V4l2CtrlH264DecodeMode) -> Self { + SafeExtControl::::from_value(decode_mode as i32) + } +} diff --git a/src/device/v4l2/stateless/device.rs b/src/device/v4l2/stateless/device.rs new file mode 100644 index 00000000..08a15f1b --- /dev/null +++ b/src/device/v4l2/stateless/device.rs @@ -0,0 +1,147 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use crate::device::v4l2::stateless::queue::V4l2CaptureBuffer; +use crate::device::v4l2::stateless::queue::V4l2CaptureQueue; +use crate::device::v4l2::stateless::queue::V4l2OutputBuffer; +use crate::device::v4l2::stateless::queue::V4l2OutputQueue; +use crate::device::v4l2::stateless::request::V4l2Request; +use crate::Resolution; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::os::fd::AsRawFd; +use std::os::fd::RawFd; +use std::path::Path; +use std::rc::Rc; +use std::sync::Arc; + +use v4l2r::device::Device as VideoDevice; +use v4l2r::device::DeviceConfig; +use v4l2r::ioctl; +use v4l2r::nix::fcntl::open; +use v4l2r::nix::fcntl::OFlag; +use v4l2r::nix::sys::stat::Mode; + +//TODO: handle memory backends other than mmap +//TODO: handle video formats other than h264 +//TODO: handle queue start/stop at runtime +//TODO: handle DRC at runtime +struct DeviceHandle { + video_device: Arc, + media_device: RawFd, + output_queue: V4l2OutputQueue, + capture_queue: V4l2CaptureQueue, + capture_buffers: HashMap, +} + +impl DeviceHandle { + fn new() -> Self { + // TODO: pass video device path and config via function arguments + let video_device_path = Path::new("/dev/video-dec0"); + let video_device_config = DeviceConfig::new().non_blocking_dqbuf(); + let video_device = Arc::new( + VideoDevice::open(video_device_path, video_device_config) + .expect("Failed to open video device"), + ); + // TODO: probe capabilties to find releted media device path + let media_device_path = Path::new("/dev/media-dec0"); + let media_device = open( + media_device_path, + OFlag::O_RDWR | OFlag::O_CLOEXEC, + Mode::empty(), + ) + .unwrap_or_else(|_| panic!("Cannot open {}", media_device_path.display())); + // TODO: handle custom configuration + const NUM_OUTPUT_BUFFERS: u32 = 8; + const NUM_CAPTURE_BUFFERS: u32 = 8; + let output_queue = V4l2OutputQueue::new(video_device.clone(), NUM_OUTPUT_BUFFERS); + let capture_queue = V4l2CaptureQueue::new(video_device.clone(), NUM_CAPTURE_BUFFERS); + Self { + video_device, + media_device, + output_queue, + capture_queue, + capture_buffers: HashMap::::new(), + } + } + fn alloc_request(&self) -> ioctl::Request { + ioctl::Request::alloc(&self.media_device).expect("Failed to alloc request handle") + } + fn alloc_buffer(&self) -> V4l2OutputBuffer { + self.output_queue.alloc_buffer() + } + fn sync(&mut self, timestamp: u64) -> V4l2CaptureBuffer { + // TODO: handle synced buffers internally by capture queue + loop { + match self.capture_buffers.remove(×tamp) { + Some(buffer) => return buffer, + _ => self.recycle_buffers(), // TODO: poll/select + }; + } + } + fn recycle_buffers(&mut self) { + self.output_queue.drain(); + // TODO: handle synced buffers internally by capture queue + loop { + match self.capture_queue.dequeue_buffer() { + Some(buffer) => { + self.capture_buffers.insert(buffer.timestamp(), buffer); + } + _ => break, + } + } + self.capture_queue.refill(); + } +} + +#[derive(Clone)] +pub struct V4l2Device { + handle: Rc>, +} + +impl V4l2Device { + pub fn new() -> Self { + Self { + handle: Rc::new(RefCell::new(DeviceHandle::new())), + } + } + pub fn num_free_buffers(&self) -> usize { + self.handle.borrow().output_queue.num_free_buffers() + } + pub fn num_buffers(&self) -> usize { + self.handle.borrow().output_queue.num_buffers() + } + pub fn set_resolution(&mut self, resolution: Resolution) -> &mut Self { + self.handle + .borrow_mut() + .output_queue + .set_resolution(resolution); + self.handle + .borrow_mut() + .capture_queue + .set_resolution(resolution); + self + } + pub fn alloc_request(&self, timestamp: u64) -> V4l2Request { + V4l2Request::new( + self.clone(), + timestamp, + self.handle.borrow().alloc_request(), + self.handle.borrow().alloc_buffer(), + ) + } + pub fn sync(&self, timestamp: u64) -> V4l2CaptureBuffer { + self.handle.borrow_mut().sync(timestamp) + } + pub fn recycle_buffers(&self) { + self.handle.borrow_mut().recycle_buffers() + } +} + +impl AsRawFd for V4l2Device { + fn as_raw_fd(&self) -> i32 { + self.handle.borrow().video_device.as_raw_fd() + } +} diff --git a/src/device/v4l2/stateless/queue.rs b/src/device/v4l2/stateless/queue.rs new file mode 100644 index 00000000..ce878f21 --- /dev/null +++ b/src/device/v4l2/stateless/queue.rs @@ -0,0 +1,321 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +use v4l2r::bindings::v4l2_format; +use v4l2r::device::queue::direction::Capture; +use v4l2r::device::queue::direction::Output; +use v4l2r::device::queue::dqbuf::DqBuffer; +use v4l2r::device::queue::qbuf::get_free::GetFreeCaptureBuffer; +use v4l2r::device::queue::qbuf::get_free::GetFreeOutputBuffer; +use v4l2r::device::queue::qbuf::weak::QBufferWeak; +use v4l2r::device::queue::BuffersAllocated; +use v4l2r::device::queue::Queue; +use v4l2r::device::queue::QueueInit; +use v4l2r::device::AllocatedQueue; +use v4l2r::device::Device; +use v4l2r::device::Stream; +use v4l2r::device::TryDequeue; +use v4l2r::memory::MemoryType; +use v4l2r::memory::MmapHandle; +use v4l2r::nix::sys::time::TimeVal; +use v4l2r::Format; +use v4l2r::PixelFormat; +use v4l2r::PlaneLayout; + +use crate::Resolution; + +//TODO: handle memory backends other than mmap +pub struct V4l2OutputBuffer { + queue: V4l2OutputQueue, + handle: QBufferWeak, Vec>, + length: usize, +} + +impl V4l2OutputBuffer { + fn new(queue: V4l2OutputQueue, handle: QBufferWeak, Vec>) -> Self { + Self { + queue, + handle, + length: 0, + } + } + pub fn index(&self) -> usize { + self.handle.index() + } + pub fn length(&self) -> usize { + self.length + } + pub fn write(&mut self, data: &[u8]) -> &mut Self { + let mut mapping = self + .handle + .get_plane_mapping(0) + .expect("Failed to mmap output buffer"); + + mapping.as_mut()[self.length..self.length + 3].copy_from_slice(&[0, 0, 1]); + self.length += 3; + + mapping.as_mut()[self.length..self.length + data.len()].copy_from_slice(data); + self.length += data.len(); + + drop(mapping); + self + } + pub fn submit(self, timestamp: u64, request_fd: i32) { + let handle = &*self.queue.handle.borrow(); + let queue = match handle { + V4l2OutputQueueHandle::Streaming(queue) => queue, + _ => panic!("ERROR"), + }; + self.handle + .set_timestamp(TimeVal::new(/* FIXME: sec */ 0, timestamp as i64)) + .set_request(request_fd) + .queue(&[self.length], queue) + .expect("Failed to queue output buffer"); + } +} + +//TODO: handle memory backends other than mmap +//TODO: handle video formats other than h264 +//TODO: handle queue start/stop at runtime +//TODO: handle DRC at runtime +#[derive(Default)] +enum V4l2OutputQueueHandle { + Init(Queue), + Streaming(Queue>>), + #[default] + Unknown, +} + +#[derive(Clone)] +pub struct V4l2OutputQueue { + handle: Rc>, + num_buffers: u32, +} + +impl V4l2OutputQueue { + pub fn new(device: Arc, num_buffers: u32) -> Self { + let handle = Queue::get_output_mplane_queue(device).expect("Failed to get output queue"); + println!("Output queue:\n\tstate: None -> Init\n"); + let handle = Rc::new(RefCell::new(V4l2OutputQueueHandle::Init(handle))); + Self { + handle, + num_buffers, + } + } + pub fn set_resolution(&mut self, res: Resolution) -> &mut Self { + self.handle.replace(match self.handle.take() { + V4l2OutputQueueHandle::Init(mut handle) => { + let (width, height) = res.into(); + + handle.change_format() + .expect("Failed to change output format") + .set_size(width as usize, height as usize) + .set_pixelformat(PixelFormat::from_fourcc(b"S264")) + // 1 MB per decoding unit should be enough for most streams. + .set_planes_layout(vec![PlaneLayout { + sizeimage: 1024 * 1024, + ..Default::default() + }]) + .apply::() + .expect("Failed to apply output format"); + + let format: Format = handle.get_format() + .expect("Failed to get output format"); + println!( + "Output format:\n\t{:?}\n", format + ); + + let handle = handle + .request_buffers_generic::>(MemoryType::Mmap, self.num_buffers) + .expect("Failed to request output buffers"); + println!( + "Output queue:\n\tnum_buffers: {}\n\tnum_queued_buffers: {}\n\tnum_free_buffers: {}\n", + handle.num_buffers(), handle.num_queued_buffers(), handle.num_free_buffers() + ); + + // TODO: handle start/stop at runtime + handle.stream_on() + .expect("Failed to start output queue"); + + println!("Output queue:\n\tstate: Init -> Streaming\n"); + V4l2OutputQueueHandle::Streaming(handle) + }, + _ => { + /* TODO: handle DRC */ + todo!() + } + }); + self + } + pub fn num_buffers(&self) -> usize { + let handle = &*self.handle.borrow(); + match handle { + V4l2OutputQueueHandle::Streaming(handle) => handle.num_buffers(), + _ => 0, + } + } + pub fn num_free_buffers(&self) -> usize { + let handle = &*self.handle.borrow(); + match handle { + V4l2OutputQueueHandle::Streaming(handle) => handle.num_free_buffers(), + _ => 0, + } + } + pub fn alloc_buffer(&self) -> V4l2OutputBuffer { + let handle = &*self.handle.borrow(); + match handle { + V4l2OutputQueueHandle::Streaming(handle) => V4l2OutputBuffer::new( + self.clone(), + handle + .try_get_free_buffer() + .expect("Failed to alloc output buffer") + .take(), + ), + _ => panic!("ERROR"), + } + } + pub fn drain(&self) { + let handle = &*self.handle.borrow(); + match handle { + V4l2OutputQueueHandle::Streaming(handle) => loop { + match handle.try_dequeue() { + Ok(buffer) => continue, + _ => break, + } + }, + _ => panic!("ERROR"), + } + } +} + +// TODO: handle other memory backends +pub struct V4l2CaptureBuffer { + handle: DqBuffer>, +} + +impl V4l2CaptureBuffer { + fn new(handle: DqBuffer>) -> Self { + Self { handle } + } + pub fn index(&self) -> usize { + self.handle.data.index() as usize + } + pub fn timestamp(&self) -> u64 { + self.handle.data.timestamp().tv_usec as u64 + } + pub fn length(&self) -> usize { + let mut length = 0; + for i in 0..self.handle.data.num_planes() { + let mapping = self + .handle + .get_plane_mapping(i) + .expect("Failed to mmap capture buffer"); + length += mapping.size(); + drop(mapping); + } + length + } + pub fn read(&self, data: &mut [u8]) { + let mut offset = 0; + for i in 0..self.handle.data.num_planes() { + let mapping = self + .handle + .get_plane_mapping(i) + .expect("Failed to mmap capture buffer"); + data[offset..offset + mapping.size()].copy_from_slice(&mapping); + offset += mapping.size(); + drop(mapping); + } + } +} + +//TODO: handle memory backends other than mmap +//TODO: handle video formats other than h264 +//TODO: handle queue start/stop at runtime +//TODO: handle DRC at runtime +//TODO: handle synced buffers in Streaming state +#[derive(Default)] +enum V4l2CaptureQueueHandle { + Init(Queue), + Streaming(Queue>>), + #[default] + Unknown, +} + +pub struct V4l2CaptureQueue { + handle: RefCell, + num_buffers: u32, +} + +impl V4l2CaptureQueue { + pub fn new(device: Arc, num_buffers: u32) -> Self { + let handle = Queue::get_capture_mplane_queue(device).expect("Failed to get capture queue"); + println!("Capture queue:\n\tstate: None -> Init\n"); + let handle = RefCell::new(V4l2CaptureQueueHandle::Init(handle)); + Self { + handle, + num_buffers, + } + } + pub fn set_resolution(&mut self, _: Resolution) -> &mut Self { + self.handle.replace(match self.handle.take() { + V4l2CaptureQueueHandle::Init(handle) => { + let format: Format = handle.get_format() + .expect("Failed to get capture format"); + println!( + "Capture format:\n\t{:?}\n", format + ); + + let handle = handle + .request_buffers_generic::>(MemoryType::Mmap, self.num_buffers) + .expect("Failed to request capture buffers"); + println!( + "Capture queue:\n\tnum_buffers: {}\n\tnum_queued_buffers: {}\n\tnum_free_buffers: {}\n", + handle.num_buffers(), handle.num_queued_buffers(), handle.num_free_buffers() + ); + + // TODO: handle start/stop at runtime + handle.stream_on() + .expect("Failed to start capture queue"); + + println!("Capture queue:\n\tstate: Init -> Streaming\n"); + V4l2CaptureQueueHandle::Streaming(handle) + }, + _ => { + /* TODO: handle DRC */ + todo!() + } + }); + self + } + pub fn dequeue_buffer(&self) -> Option { + let handle = &*self.handle.borrow(); + match handle { + V4l2CaptureQueueHandle::Streaming(handle) => match handle.try_dequeue() { + Ok(buffer) => Some(V4l2CaptureBuffer::new(buffer)), + _ => None, + }, + _ => panic!("ERROR"), + } + } + pub fn refill(&self) { + let handle = &*self.handle.borrow(); + match handle { + V4l2CaptureQueueHandle::Streaming(handle) => { + while handle.num_free_buffers() != 0 { + handle + .try_get_free_buffer() + .expect("Failed to alloc capture buffer") + .queue() + .expect("Failed to queue capture buffer"); + } + } + _ => panic!("ERROR"), + } + } +} diff --git a/src/device/v4l2/stateless/request.rs b/src/device/v4l2/stateless/request.rs new file mode 100644 index 00000000..31c244a6 --- /dev/null +++ b/src/device/v4l2/stateless/request.rs @@ -0,0 +1,206 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::cell::RefCell; +use std::os::fd::AsRawFd; +use std::rc::Rc; + +use v4l2r::controls::ExtControlTrait; +use v4l2r::controls::SafeExtControl; +use v4l2r::ioctl; + +use crate::device::v4l2::stateless::device::V4l2Device; +use crate::device::v4l2::stateless::queue::V4l2CaptureBuffer; +use crate::device::v4l2::stateless::queue::V4l2OutputBuffer; + +struct InitRequestHandle { + device: V4l2Device, + timestamp: u64, + handle: ioctl::Request, + buffer: V4l2OutputBuffer, +} + +impl InitRequestHandle { + fn new( + device: V4l2Device, + timestamp: u64, + handle: ioctl::Request, + buffer: V4l2OutputBuffer, + ) -> Self { + Self { + device, + timestamp, + handle, + buffer, + } + } + fn ioctl(&mut self, ctrl: C) -> &mut Self + where + C: Into>, + T: ExtControlTrait, + { + let which = ioctl::CtrlWhich::Request(self.handle.as_raw_fd()); + let mut ctrl: SafeExtControl = ctrl.into(); + ioctl::s_ext_ctrls(&self.device, which, &mut ctrl).expect("Failed to set output control"); + self + } + fn write(&mut self, data: &[u8]) -> &mut Self { + self.buffer.write(data); + self + } + fn submit(self) -> PendingRequestHandle { + self.buffer.submit(self.timestamp, self.handle.as_raw_fd()); + self.handle.queue().expect("Failed to queue request handle"); + PendingRequestHandle { + device: self.device.clone(), + timestamp: self.timestamp, + } + } +} + +struct PendingRequestHandle { + device: V4l2Device, + timestamp: u64, +} + +impl PendingRequestHandle { + fn sync(self) -> DoneRequestHandle { + DoneRequestHandle { + buffer: Rc::new(RefCell::new(self.device.sync(self.timestamp))), + } + } +} + +struct DoneRequestHandle { + buffer: Rc>, +} + +impl DoneRequestHandle { + fn result(&self) -> V4l2Result { + V4l2Result { + buffer: self.buffer.clone(), + } + } +} + +#[derive(Default)] +enum RequestHandle { + Init(InitRequestHandle), + Pending(PendingRequestHandle), + Done(DoneRequestHandle), + #[default] + Unknown, +} + +impl RequestHandle { + fn new( + device: V4l2Device, + timestamp: u64, + handle: ioctl::Request, + buffer: V4l2OutputBuffer, + ) -> Self { + Self::Init(InitRequestHandle::new(device, timestamp, handle, buffer)) + } + fn timestamp(&self) -> u64 { + match self { + Self::Init(handle) => handle.timestamp, + Self::Pending(handle) => handle.timestamp, + Self::Done(handle) => handle.buffer.borrow().timestamp(), + _ => panic!("ERROR"), + } + } + fn ioctl(&mut self, ctrl: C) -> &mut Self + where + C: Into>, + T: ExtControlTrait, + { + match self { + Self::Init(handle) => handle.ioctl(ctrl), + _ => panic!("ERROR"), + }; + self + } + fn write(&mut self, data: &[u8]) -> &mut Self { + match self { + Self::Init(handle) => handle.write(data), + _ => panic!("ERROR"), + }; + self + } + fn submit(self) -> Self { + match self { + Self::Init(handle) => Self::Pending(handle.submit()), + _ => panic!("ERROR"), + } + } + fn sync(self) -> Self { + match self { + Self::Pending(handle) => Self::Done(handle.sync()), + Self::Done(_) => self, + _ => panic!("ERROR"), + } + } + fn result(&self) -> V4l2Result { + match self { + Self::Done(handle) => handle.result(), + _ => panic!("ERROR"), + } + } +} + +pub struct V4l2Request { + handle: RefCell, +} + +impl V4l2Request { + pub fn new( + device: V4l2Device, + timestamp: u64, + handle: ioctl::Request, + buffer: V4l2OutputBuffer, + ) -> Self { + Self { + handle: RefCell::new(RequestHandle::new(device, timestamp, handle, buffer)), + } + } + pub fn timestamp(&self) -> u64 { + self.handle.borrow().timestamp() + } + pub fn ioctl(&self, ctrl: C) -> &Self + where + C: Into>, + T: ExtControlTrait, + { + self.handle.borrow_mut().ioctl(ctrl); + self + } + pub fn write(&self, data: &[u8]) -> &Self { + self.handle.borrow_mut().write(data); + self + } + pub fn submit(&self) -> &Self { + self.handle.replace(self.handle.take().submit()); + self + } + pub fn sync(&self) -> &Self { + self.handle.replace(self.handle.take().sync()); + self + } + pub fn result(&self) -> V4l2Result { + self.handle.borrow().result() + } +} + +pub struct V4l2Result { + buffer: Rc>, +} + +impl V4l2Result { + pub fn length(&self) -> usize { + self.buffer.borrow().length() + } + pub fn read(&self, data: &mut [u8]) { + self.buffer.borrow().read(data) + } +} diff --git a/src/lib.rs b/src/lib.rs index b20a60d7..502664cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,7 @@ pub mod backend; pub mod codec; pub mod decoder; +pub mod device; pub mod encoder; pub mod utils; From 52f81bd4ad3b8e47af522e9a601ba58c1216f0f1 Mon Sep 17 00:00:00 2001 From: Slawomir Rosek Date: Thu, 6 Jun 2024 08:17:55 +0200 Subject: [PATCH 3/7] backend/v4l2: Add v4l2 stateless decoder backend Initial implementation of v4l2 stateless decoder backend. --- src/backend/v4l2.rs | 3 +- src/backend/v4l2/decoder.rs | 5 + src/backend/v4l2/decoder/stateless.rs | 191 ++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 src/backend/v4l2/decoder.rs create mode 100644 src/backend/v4l2/decoder/stateless.rs diff --git a/src/backend/v4l2.rs b/src/backend/v4l2.rs index c607540e..9227ee78 100644 --- a/src/backend/v4l2.rs +++ b/src/backend/v4l2.rs @@ -2,8 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -//! V4L2 backend for stateful encoders. +//! V4L2 backend +pub mod decoder; pub mod encoder; impl From for crate::Fourcc { diff --git a/src/backend/v4l2/decoder.rs b/src/backend/v4l2/decoder.rs new file mode 100644 index 00000000..9ce03c2e --- /dev/null +++ b/src/backend/v4l2/decoder.rs @@ -0,0 +1,5 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +pub mod stateless; diff --git a/src/backend/v4l2/decoder/stateless.rs b/src/backend/v4l2/decoder/stateless.rs new file mode 100644 index 00000000..036fd4b6 --- /dev/null +++ b/src/backend/v4l2/decoder/stateless.rs @@ -0,0 +1,191 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::cell::RefCell; +use std::rc::Rc; + +use crate::decoder::stateless::PoolLayer; +use crate::decoder::stateless::StatelessCodec; +use crate::decoder::stateless::StatelessDecoderBackend; +use crate::decoder::stateless::TryFormat; +use crate::decoder::DecodedHandle; +use crate::decoder::DynHandle; +use crate::decoder::FramePool; +use crate::decoder::MappableHandle; +use crate::decoder::StreamInfo; +use crate::DecodedFormat; +use crate::Resolution; + +use crate::device::v4l2::stateless::device::V4l2Device; +use crate::device::v4l2::stateless::request::V4l2Request; + +pub struct V4l2Picture { + request: V4l2Request, + // To properly decode stream while output and capture queues + // are processed independently it's required for v4l2 backend + // to maintain DPB buffer recycling. The following vector + // is used to prevent reference pictures to be reused while + // current picture is still being decoded. + // TODO: handle ref list inernally by V4l2Request. + ref_pictures: Option>>>, +} + +impl V4l2Picture { + pub fn new(request: V4l2Request) -> Self { + Self { + request, + ref_pictures: None, + } + } + pub fn timestamp(&self) -> u64 { + self.request.timestamp() + } + pub fn set_ref_pictures(&mut self, ref_pictures: Vec>>) -> &mut Self { + self.ref_pictures = Some(ref_pictures); + self + } + pub fn sync(&mut self) -> &mut Self { + self.request.sync(); + self.ref_pictures = None; + self + } + pub fn request(&self) -> &V4l2Request { + &self.request + } +} + +impl<'a> MappableHandle for std::cell::Ref<'a, V4l2Picture> { + fn read(&mut self, data: &mut [u8]) -> anyhow::Result<()> { + self.request.result().read(data); + Ok(()) + } + fn image_size(&mut self) -> usize { + self.request.result().length() + } +} + +pub struct BackendHandle { + pub picture: Rc>, +} + +impl<'a> DynHandle for std::cell::Ref<'a, BackendHandle> { + fn dyn_mappable_handle<'b>(&'b self) -> anyhow::Result> { + self.picture.borrow_mut().sync(); + Ok(Box::new(self.picture.borrow())) + } +} + +pub struct V4l2StatelessDecoderHandle { + pub handle: Rc>, +} + +impl Clone for V4l2StatelessDecoderHandle { + fn clone(&self) -> Self { + Self { + handle: Rc::clone(&self.handle), + } + } +} + +impl DecodedHandle for V4l2StatelessDecoderHandle { + type Descriptor = (); + + fn coded_resolution(&self) -> Resolution { + todo!(); + } + + fn display_resolution(&self) -> Resolution { + todo!(); + } + + fn timestamp(&self) -> u64 { + self.handle.borrow().picture.borrow().timestamp() + } + + fn dyn_picture<'a>(&'a self) -> Box { + Box::new(self.handle.borrow()) + } + + fn sync(&self) -> anyhow::Result<()> { + Ok(()) + } + + fn is_ready(&self) -> bool { + todo!(); + } + + fn resource(&self) -> std::cell::Ref<()> { + todo!(); + } +} + +pub struct V4l2StatelessDecoderBackend { + pub device: V4l2Device, + stream_info: StreamInfo, +} + +impl V4l2StatelessDecoderBackend { + pub fn new() -> Self { + Self { + device: V4l2Device::new(), + stream_info: StreamInfo { + format: DecodedFormat::I420, + min_num_frames: 4, + coded_resolution: Resolution::from((320, 200)), + display_resolution: Resolution::from((320, 200)), + }, + } + } +} + +impl FramePool for V4l2StatelessDecoderBackend { + type Descriptor = (); + + fn coded_resolution(&self) -> Resolution { + todo!(); + } + + fn set_coded_resolution(&mut self, _resolution: Resolution) { + todo!(); + } + + fn add_frames(&mut self, _descriptors: Vec) -> Result<(), anyhow::Error> { + todo!(); + } + + fn num_free_frames(&self) -> usize { + self.device.num_free_buffers() + } + + fn num_managed_frames(&self) -> usize { + self.device.num_buffers() + } + + fn clear(&mut self) { + todo!(); + } +} + +impl TryFormat for V4l2StatelessDecoderBackend { + fn try_format(&mut self, _: &Codec::FormatInfo, _: DecodedFormat) -> anyhow::Result<()> { + // TODO + Ok(()) + } +} + +impl StatelessDecoderBackend for V4l2StatelessDecoderBackend { + type Handle = V4l2StatelessDecoderHandle; + + type FramePool = Self; + + fn stream_info(&self) -> Option<&StreamInfo> { + // TODO + Some(&self.stream_info) + } + + fn frame_pool(&mut self, _: PoolLayer) -> Vec<&mut Self::FramePool> { + self.device.recycle_buffers(); + vec![self] + } +} From 2a66965f07b57f78e4e2c90730fa4ed882bfb4f1 Mon Sep 17 00:00:00 2001 From: Slawomir Rosek Date: Thu, 20 Jun 2024 16:45:00 +0200 Subject: [PATCH 4/7] decoder/h264: Add support for v4l2 backend Initial implementation of h264 decoder on top of v4l2 stateless backend TODO: * Add support for scaling matrix --- src/decoder/stateless/h264.rs | 2 + src/decoder/stateless/h264/v4l2.rs | 140 +++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 src/decoder/stateless/h264/v4l2.rs diff --git a/src/decoder/stateless/h264.rs b/src/decoder/stateless/h264.rs index bcd0a88c..5b855557 100644 --- a/src/decoder/stateless/h264.rs +++ b/src/decoder/stateless/h264.rs @@ -4,6 +4,8 @@ #[cfg(any(test, fuzzing))] mod dummy; +#[cfg(feature = "v4l2")] +mod v4l2; #[cfg(feature = "vaapi")] mod vaapi; diff --git a/src/decoder/stateless/h264/v4l2.rs b/src/decoder/stateless/h264/v4l2.rs new file mode 100644 index 00000000..fad5418e --- /dev/null +++ b/src/decoder/stateless/h264/v4l2.rs @@ -0,0 +1,140 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::cell::RefCell; +use std::rc::Rc; + +use crate::backend::v4l2::decoder::stateless::BackendHandle; +use crate::backend::v4l2::decoder::stateless::V4l2Picture; +use crate::backend::v4l2::decoder::stateless::V4l2StatelessDecoderBackend; +use crate::backend::v4l2::decoder::stateless::V4l2StatelessDecoderHandle; +use crate::codec::h264::dpb::Dpb; +use crate::codec::h264::dpb::DpbEntry; +use crate::codec::h264::parser::Pps; +use crate::codec::h264::parser::Slice; +use crate::codec::h264::parser::SliceHeader; +use crate::codec::h264::parser::Sps; +use crate::codec::h264::picture::PictureData; +use crate::decoder::stateless::h264::StatelessH264DecoderBackend; +use crate::decoder::stateless::h264::H264; +use crate::decoder::stateless::StatelessBackendResult; +use crate::decoder::stateless::StatelessDecoder; +use crate::decoder::stateless::StatelessDecoderBackendPicture; +use crate::decoder::BlockingMode; +use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DecodeMode; +use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DecodeParams; +use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DpbEntry; +use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264Pps; +//TODO use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264ScalingMatrix; +use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264Sps; +use crate::Resolution; + +impl StatelessDecoderBackendPicture for V4l2StatelessDecoderBackend { + type Picture = Rc>; +} + +impl StatelessH264DecoderBackend for V4l2StatelessDecoderBackend { + fn new_sequence(&mut self, sps: &Rc) -> StatelessBackendResult<()> { + let mb_unit = 16; + let map_unit = 16; + let resolution = Resolution::from(( + (sps.pic_width_in_mbs_minus1 + 1) * mb_unit, + (sps.pic_height_in_map_units_minus1 + 1) * map_unit, + )); + self.device.set_resolution(resolution); + Ok(()) + } + + fn new_picture( + &mut self, + _: &PictureData, + timestamp: u64, + ) -> StatelessBackendResult { + Ok(Rc::new(RefCell::new(V4l2Picture::new( + self.device.alloc_request(timestamp), + )))) + } + + fn new_field_picture( + &mut self, + _: &PictureData, + _: u64, + _: &Self::Handle, + ) -> StatelessBackendResult { + todo!() + } + + fn start_picture( + &mut self, + picture: &mut Self::Picture, + picture_data: &PictureData, + sps: &Sps, + pps: &Pps, + dpb: &Dpb, + slice_header: &SliceHeader, + ) -> StatelessBackendResult<()> { + let mut dpb_entries = Vec::::new(); + let mut ref_pictures = Vec::>>::new(); + for entry in dpb.entries() { + let ref_picture = match &entry.handle { + Some(handle) => handle.handle.borrow().picture.clone(), + None => todo!(), + }; + dpb_entries.push(V4l2CtrlH264DpbEntry { + timestamp: ref_picture.borrow().timestamp(), + pic: entry.pic.clone(), + }); + ref_pictures.push(ref_picture); + } + let mut h264_sps = V4l2CtrlH264Sps::new(); + let mut h264_pps = V4l2CtrlH264Pps::new(); + //TODO let mut h264_scaling_matrix = V4l2CtrlH264ScalingMatrix::new(); + let mut h264_decode_params = V4l2CtrlH264DecodeParams::new(); + h264_sps.set(sps); + h264_pps.set(pps); + h264_decode_params + .set_picture_data(picture_data) + .set_dpb_entries(dpb_entries) + .set_slice_header(slice_header); + let mut picture = picture.borrow_mut(); + picture + .request() + .ioctl(&h264_sps) + .ioctl(&h264_pps) + //TODO.ioctl(&h264_scaling_matrix) + .ioctl(&h264_decode_params) + .ioctl(V4l2CtrlH264DecodeMode::FrameBased); + picture.set_ref_pictures(ref_pictures); + Ok(()) + } + + fn decode_slice( + &mut self, + picture: &mut Self::Picture, + slice: &Slice, + _: &Sps, + _: &Pps, + _: &[&DpbEntry], + _: &[&DpbEntry], + ) -> StatelessBackendResult<()> { + picture.borrow().request().write(slice.nalu.as_ref()); + Ok(()) + } + + fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult { + let handle = Rc::new(RefCell::new(BackendHandle { + picture: picture.clone(), + })); + picture.borrow().request().submit(); + Ok(V4l2StatelessDecoderHandle { handle }) + } +} + +impl StatelessDecoder { + // Creates a new instance of the decoder using the v4l2 backend. + pub fn new_v4l2(blocking_mode: BlockingMode) -> Self { + Self::new(V4l2StatelessDecoderBackend::new(), blocking_mode) + .expect("Failed to create v4l2 stateless decoder backend") + } +} From 07dfccd7ac991c6fd081522bba9ff577a773024f Mon Sep 17 00:00:00 2001 From: Slawomir Rosek Date: Thu, 6 Jun 2024 16:52:10 +0200 Subject: [PATCH 5/7] examples/ccdec: Add example for v4l2 backend --- Cargo.toml | 4 + examples/ccdec-v4l2-stateless.rs | 187 +++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 examples/ccdec-v4l2-stateless.rs diff --git a/Cargo.toml b/Cargo.toml index e620be93..070671ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,10 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } name = "ccdec" required-features = ["vaapi"] +[[example]] +name = "ccdec-v4l2-stateless" +required-features = ["v4l2"] + [[example]] name = "ccenc" required-features = ["vaapi"] diff --git a/examples/ccdec-v4l2-stateless.rs b/examples/ccdec-v4l2-stateless.rs new file mode 100644 index 00000000..69891ee8 --- /dev/null +++ b/examples/ccdec-v4l2-stateless.rs @@ -0,0 +1,187 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::borrow::Cow; +use std::fs::File; +use std::io::Read; +use std::io::Write; +use std::path::PathBuf; +use std::str::FromStr; + +use argh::FromArgs; +use cros_codecs::backend::v4l2::decoder::stateless::V4l2StatelessDecoderHandle; +use cros_codecs::codec::h264::parser::Nalu as H264Nalu; +use cros_codecs::decoder::stateless::h264::H264; +use cros_codecs::decoder::stateless::StatelessDecoder; +use cros_codecs::decoder::stateless::StatelessVideoDecoder; +use cros_codecs::decoder::BlockingMode; +use cros_codecs::decoder::DecodedHandle; +use cros_codecs::multiple_desc_type; +use cros_codecs::utils::simple_playback_loop; +use cros_codecs::utils::simple_playback_loop_owned_frames; +use cros_codecs::utils::DmabufFrame; +use cros_codecs::utils::NalIterator; +use cros_codecs::utils::UserPtrFrame; +use cros_codecs::DecodedFormat; + +multiple_desc_type! { + enum BufferDescriptor { + Managed(()), + Dmabuf(DmabufFrame), + User(UserPtrFrame), + } +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +enum EncodedFormat { + H264, + H265, + VP8, + VP9, + AV1, +} + +impl FromStr for EncodedFormat { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "h264" | "H264" => Ok(EncodedFormat::H264), + "h265" | "H265" => Ok(EncodedFormat::H265), + "vp8" | "VP8" => Ok(EncodedFormat::VP8), + "vp9" | "VP9" => Ok(EncodedFormat::VP9), + "av1" | "AV1" => Ok(EncodedFormat::AV1), + _ => Err("unrecognized input format. Valid values: h264, h265, vp8, vp9, av1"), + } + } +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +enum FrameMemoryType { + Managed, + Prime, + User, +} + +impl FromStr for FrameMemoryType { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "managed" => Ok(FrameMemoryType::Managed), + "prime" => Ok(FrameMemoryType::Prime), + "user" => Ok(FrameMemoryType::User), + _ => Err("unrecognized memory type. Valid values: managed, prime, user"), + } + } +} + +/// Simple player using cros-codecs +#[derive(Debug, FromArgs)] +struct Args { + /// input file + #[argh(positional)] + input: PathBuf, + + /// output file to write the decoded frames to + #[argh(option)] + output: Option, + + /// input format to decode from. + #[argh(option)] + input_format: EncodedFormat, + + //TODO /// pixel format to decode into. Default: i420 + //TODO #[argh(option, default = "DecodedFormat::I420")] + //TODO output_format: DecodedFormat, + /// origin of the memory for decoded buffers (managed, prime or user). Default: managed. + #[argh(option, default = "FrameMemoryType::Managed")] + frame_memory: FrameMemoryType, + + //TODO /// path to the GBM device to use if frame-memory=prime + //TODO #[argh(option)] + //TODO gbm_device: Option, + /// whether to decode frames synchronously + #[argh(switch)] + synchronous: bool, + //TODO /// whether to display the MD5 of the decoded stream, and at which granularity (stream or + //TODO /// frame) + //TODO #[argh(option)] + //TODO compute_md5: Option, +} + +fn main() { + env_logger::init(); + + let args: Args = argh::from_env(); + + let input = { + let mut buf = Vec::new(); + File::open(args.input) + .expect("error opening input file") + .read_to_end(&mut buf) + .expect("error reading input file"); + buf + }; + + let mut output = args + .output + .as_ref() + .map(|p| File::create(p).expect("Failed to create output file")); + + let blocking_mode = if args.synchronous { + todo!() // BlockingMode::Blocking + } else { + BlockingMode::NonBlocking + }; + + let (mut decoder, frame_iter) = match args.input_format { + EncodedFormat::H264 => { + let frame_iter = Box::new(NalIterator::::new(&input)) + as Box>>; + + let decoder = Box::new(StatelessDecoder::::new_v4l2(blocking_mode)) + as Box>; + + (decoder, frame_iter) + } + EncodedFormat::VP8 => todo!(), + EncodedFormat::VP9 => todo!(), + EncodedFormat::H265 => todo!(), + EncodedFormat::AV1 => todo!(), + }; + + let mut on_new_frame = |handle: V4l2StatelessDecoderHandle| { + let picture = handle.dyn_picture(); + let mut handle = picture.dyn_mappable_handle().unwrap(); + let buffer_size = handle.image_size(); + let mut frame_data = vec![0; buffer_size]; + handle.read(&mut frame_data).unwrap(); + if let Some(output) = &mut output { + output + .write_all(&frame_data) + .expect("Failed to write output file"); + } + }; + + simple_playback_loop( + decoder.as_mut(), + frame_iter, + &mut on_new_frame, + &mut |stream_info, nb_frames| { + Ok(match args.frame_memory { + FrameMemoryType::Managed => { + simple_playback_loop_owned_frames(stream_info, nb_frames)? + .into_iter() + .collect() + } + FrameMemoryType::Prime => todo!(), + FrameMemoryType::User => todo!(), + }) + }, + DecodedFormat::NV12, + blocking_mode, + ) + .expect("error during playback loop"); +} From 1af2ce1a7b46201dbe0c30017d219e3d1e4d9219 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 8 Jul 2024 15:18:23 +0900 Subject: [PATCH 6/7] decoder/stateless/v4l2/h264: remove unneeded `V4l2CtrlH264Sps` and `V4l2CtrlH264Pps` wrapper types We can achieve the same result through existing `From` implementations. --- src/decoder/stateless/h264/v4l2.rs | 18 +++++---- src/device/v4l2/stateless/controls/h264.rs | 44 ---------------------- 2 files changed, 10 insertions(+), 52 deletions(-) diff --git a/src/decoder/stateless/h264/v4l2.rs b/src/decoder/stateless/h264/v4l2.rs index fad5418e..7321b63d 100644 --- a/src/decoder/stateless/h264/v4l2.rs +++ b/src/decoder/stateless/h264/v4l2.rs @@ -5,6 +5,12 @@ use std::cell::RefCell; use std::rc::Rc; +use v4l2r::bindings::v4l2_ctrl_h264_pps; +use v4l2r::bindings::v4l2_ctrl_h264_sps; +use v4l2r::controls::codec::H264Pps; +use v4l2r::controls::codec::H264Sps; +use v4l2r::controls::SafeExtControl; + use crate::backend::v4l2::decoder::stateless::BackendHandle; use crate::backend::v4l2::decoder::stateless::V4l2Picture; use crate::backend::v4l2::decoder::stateless::V4l2StatelessDecoderBackend; @@ -25,9 +31,7 @@ use crate::decoder::BlockingMode; use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DecodeMode; use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DecodeParams; use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264DpbEntry; -use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264Pps; //TODO use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264ScalingMatrix; -use crate::device::v4l2::stateless::controls::h264::V4l2CtrlH264Sps; use crate::Resolution; impl StatelessDecoderBackendPicture for V4l2StatelessDecoderBackend { @@ -87,12 +91,10 @@ impl StatelessH264DecoderBackend for V4l2StatelessDecoderBackend { }); ref_pictures.push(ref_picture); } - let mut h264_sps = V4l2CtrlH264Sps::new(); - let mut h264_pps = V4l2CtrlH264Pps::new(); //TODO let mut h264_scaling_matrix = V4l2CtrlH264ScalingMatrix::new(); let mut h264_decode_params = V4l2CtrlH264DecodeParams::new(); - h264_sps.set(sps); - h264_pps.set(pps); + let h264_sps = SafeExtControl::::from(v4l2_ctrl_h264_sps::from(sps)); + let h264_pps = SafeExtControl::::from(v4l2_ctrl_h264_pps::from(pps)); h264_decode_params .set_picture_data(picture_data) .set_dpb_entries(dpb_entries) @@ -100,8 +102,8 @@ impl StatelessH264DecoderBackend for V4l2StatelessDecoderBackend { let mut picture = picture.borrow_mut(); picture .request() - .ioctl(&h264_sps) - .ioctl(&h264_pps) + .ioctl(h264_sps) + .ioctl(h264_pps) //TODO.ioctl(&h264_scaling_matrix) .ioctl(&h264_decode_params) .ioctl(V4l2CtrlH264DecodeMode::FrameBased); diff --git a/src/device/v4l2/stateless/controls/h264.rs b/src/device/v4l2/stateless/controls/h264.rs index 0aca98c0..f29f7ab1 100644 --- a/src/device/v4l2/stateless/controls/h264.rs +++ b/src/device/v4l2/stateless/controls/h264.rs @@ -48,9 +48,7 @@ use v4l2r::bindings::V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; use v4l2r::bindings::V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; use v4l2r::controls::codec::H264DecodeMode; use v4l2r::controls::codec::H264DecodeParams; -use v4l2r::controls::codec::H264Pps; use v4l2r::controls::codec::H264ScalingMatrix; -use v4l2r::controls::codec::H264Sps; use v4l2r::controls::SafeExtControl; impl From<&Sps> for v4l2_ctrl_h264_sps { @@ -201,48 +199,6 @@ impl From<&V4l2CtrlH264DpbEntry> for v4l2_h264_dpb_entry { } } -#[derive(Default)] -pub struct V4l2CtrlH264Sps { - handle: v4l2_ctrl_h264_sps, -} - -impl V4l2CtrlH264Sps { - pub fn new() -> Self { - Default::default() - } - pub fn set(&mut self, sps: &Sps) -> &mut Self { - self.handle = v4l2_ctrl_h264_sps::from(sps); - self - } -} - -impl From<&V4l2CtrlH264Sps> for SafeExtControl { - fn from(sps: &V4l2CtrlH264Sps) -> Self { - SafeExtControl::::from(sps.handle) - } -} - -#[derive(Default)] -pub struct V4l2CtrlH264Pps { - handle: v4l2_ctrl_h264_pps, -} - -impl V4l2CtrlH264Pps { - pub fn new() -> Self { - Default::default() - } - pub fn set(&mut self, pps: &Pps) -> &mut Self { - self.handle = v4l2_ctrl_h264_pps::from(pps); - self - } -} - -impl From<&V4l2CtrlH264Pps> for SafeExtControl { - fn from(pps: &V4l2CtrlH264Pps) -> Self { - SafeExtControl::::from(pps.handle) - } -} - #[derive(Default)] pub struct V4l2CtrlH264ScalingMatrix { handle: v4l2_ctrl_h264_scaling_matrix, From f51face747e2928a7a027d9184db19d13af8124b Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 8 Jul 2024 15:59:31 +0900 Subject: [PATCH 7/7] decoder/stateless/v4l2: remove RefCell from V4l2Request This RefCell was there to provide interior mutability because some of the methods did not take Self as mutable. With proper mutability parameters, and a use of std::mem::take, it can be removed. --- src/backend/v4l2/decoder/stateless.rs | 4 +-- src/decoder/stateless/h264/v4l2.rs | 4 +-- src/device/v4l2/stateless/request.rs | 47 +++++++++++++-------------- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/backend/v4l2/decoder/stateless.rs b/src/backend/v4l2/decoder/stateless.rs index 036fd4b6..6319a3cd 100644 --- a/src/backend/v4l2/decoder/stateless.rs +++ b/src/backend/v4l2/decoder/stateless.rs @@ -50,8 +50,8 @@ impl V4l2Picture { self.ref_pictures = None; self } - pub fn request(&self) -> &V4l2Request { - &self.request + pub fn request(&mut self) -> &mut V4l2Request { + &mut self.request } } diff --git a/src/decoder/stateless/h264/v4l2.rs b/src/decoder/stateless/h264/v4l2.rs index 7321b63d..8ac8d7c4 100644 --- a/src/decoder/stateless/h264/v4l2.rs +++ b/src/decoder/stateless/h264/v4l2.rs @@ -120,7 +120,7 @@ impl StatelessH264DecoderBackend for V4l2StatelessDecoderBackend { _: &[&DpbEntry], _: &[&DpbEntry], ) -> StatelessBackendResult<()> { - picture.borrow().request().write(slice.nalu.as_ref()); + picture.borrow_mut().request().write(slice.nalu.as_ref()); Ok(()) } @@ -128,7 +128,7 @@ impl StatelessH264DecoderBackend for V4l2StatelessDecoderBackend { let handle = Rc::new(RefCell::new(BackendHandle { picture: picture.clone(), })); - picture.borrow().request().submit(); + picture.borrow_mut().request().submit(); Ok(V4l2StatelessDecoderHandle { handle }) } } diff --git a/src/device/v4l2/stateless/request.rs b/src/device/v4l2/stateless/request.rs index 31c244a6..c831eff6 100644 --- a/src/device/v4l2/stateless/request.rs +++ b/src/device/v4l2/stateless/request.rs @@ -128,16 +128,19 @@ impl RequestHandle { }; self } - fn submit(self) -> Self { - match self { - Self::Init(handle) => Self::Pending(handle.submit()), + + // This method can modify in-place instead of returning a new value. This removes the need for + // a RefCell in V4l2Request. + fn submit(&mut self) { + match std::mem::take(self) { + Self::Init(handle) => *self = Self::Pending(handle.submit()), _ => panic!("ERROR"), } } - fn sync(self) -> Self { - match self { - Self::Pending(handle) => Self::Done(handle.sync()), - Self::Done(_) => self, + fn sync(&mut self) { + match std::mem::take(self) { + Self::Pending(handle) => *self = Self::Done(handle.sync()), + s @ Self::Done(_) => *self = s, _ => panic!("ERROR"), } } @@ -149,9 +152,7 @@ impl RequestHandle { } } -pub struct V4l2Request { - handle: RefCell, -} +pub struct V4l2Request(RequestHandle); impl V4l2Request { pub fn new( @@ -160,35 +161,31 @@ impl V4l2Request { handle: ioctl::Request, buffer: V4l2OutputBuffer, ) -> Self { - Self { - handle: RefCell::new(RequestHandle::new(device, timestamp, handle, buffer)), - } + Self(RequestHandle::new(device, timestamp, handle, buffer)) } pub fn timestamp(&self) -> u64 { - self.handle.borrow().timestamp() + self.0.timestamp() } - pub fn ioctl(&self, ctrl: C) -> &Self + pub fn ioctl(&mut self, ctrl: C) -> &mut Self where C: Into>, T: ExtControlTrait, { - self.handle.borrow_mut().ioctl(ctrl); + self.0.ioctl(ctrl); self } - pub fn write(&self, data: &[u8]) -> &Self { - self.handle.borrow_mut().write(data); + pub fn write(&mut self, data: &[u8]) -> &mut Self { + self.0.write(data); self } - pub fn submit(&self) -> &Self { - self.handle.replace(self.handle.take().submit()); - self + pub fn submit(&mut self) { + self.0.submit(); } - pub fn sync(&self) -> &Self { - self.handle.replace(self.handle.take().sync()); - self + pub fn sync(&mut self) { + self.0.sync(); } pub fn result(&self) -> V4l2Result { - self.handle.borrow().result() + self.0.result() } }