From 5dc24a5b67b00664d3a339834972ce2f19f878d1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:43:54 +0000 Subject: [PATCH 01/26] Optimize Direct3D frame buffer handling and copying --- crates/scap-direct3d/src/lib.rs | 162 +++++++++++++++++++++-------- crates/scap-ffmpeg/src/direct3d.rs | 60 ++++++++--- 2 files changed, 163 insertions(+), 59 deletions(-) diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index ac704e934f..c7f3bcd323 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -4,8 +4,8 @@ use std::{ sync::{ - Arc, - atomic::{AtomicBool, Ordering}, + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, mpsc::RecvError, }, time::Duration, @@ -25,10 +25,9 @@ use windows::{ Direct3D::D3D_DRIVER_TYPE_HARDWARE, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_MAP_READ_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, - ID3D11DeviceContext, ID3D11Texture2D, + D3D11_CPU_ACCESS_READ, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, + ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, }, Dxgi::{ Common::{ @@ -69,6 +68,88 @@ impl PixelFormat { } } +const STAGING_POOL_SIZE: usize = 3; + +struct PooledStagingTexture { + texture: ID3D11Texture2D, + width: u32, + height: u32, +} + +pub struct StagingTexturePool { + textures: Mutex>, + d3d_device: ID3D11Device, + pixel_format: PixelFormat, + next_index: AtomicUsize, +} + +impl StagingTexturePool { + fn new(d3d_device: ID3D11Device, pixel_format: PixelFormat) -> Self { + Self { + textures: Mutex::new(Vec::with_capacity(STAGING_POOL_SIZE)), + d3d_device, + pixel_format, + next_index: AtomicUsize::new(0), + } + } + + fn get_or_create_texture( + &self, + width: u32, + height: u32, + ) -> windows::core::Result { + let mut textures = self.textures.lock().unwrap(); + + let index = self.next_index.fetch_add(1, Ordering::Relaxed) % STAGING_POOL_SIZE; + + if let Some(pooled) = textures.get(index) { + if pooled.width == width && pooled.height == height { + return Ok(pooled.texture.clone()); + } + } + + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: self.pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { + self.d3d_device + .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; + }; + + let texture = texture.unwrap(); + + if index < textures.len() { + textures[index] = PooledStagingTexture { + texture: texture.clone(), + width, + height, + }; + } else { + textures.push(PooledStagingTexture { + texture: texture.clone(), + width, + height, + }); + } + + Ok(texture) + } +} + pub fn is_supported() -> windows::core::Result { Ok(ApiInformation::IsApiContractPresentByMajor( &HSTRING::from("Windows.Foundation.UniversalApiContract"), @@ -201,6 +282,11 @@ impl Capturer { .map_err(NewCapturerError::Context)? .unwrap(); + let staging_pool = Arc::new(StagingTexturePool::new( + d3d_device.clone(), + settings.pixel_format, + )); + let item = item.clone(); let settings = settings.clone(); let stop_flag = Arc::new(AtomicBool::new(false)); @@ -273,6 +359,7 @@ impl Capturer { let d3d_context = d3d_context.clone(); let d3d_device = d3d_device.clone(); let stop_flag = stop_flag.clone(); + let staging_pool = staging_pool.clone(); move |frame_pool, _| { if stop_flag.load(Ordering::Relaxed) { @@ -312,6 +399,7 @@ impl Capturer { texture: cropped_texture, d3d_context: d3d_context.clone(), d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), } } else { Frame { @@ -322,6 +410,7 @@ impl Capturer { texture, d3d_context: d3d_context.clone(), d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), } }; @@ -407,6 +496,7 @@ pub struct Frame { texture: ID3D11Texture2D, d3d_device: ID3D11Device, d3d_context: ID3D11DeviceContext, + staging_pool: Arc, } impl std::fmt::Debug for Frame { @@ -447,49 +537,29 @@ impl Frame { &self.d3d_context } - pub fn as_buffer<'a>(&'a self) -> windows::core::Result> { - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: self.width, - Height: self.height, - MipLevels: 1, - ArraySize: 1, - Format: self.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_STAGING, - BindFlags: 0, - CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32 | D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; - - let mut texture = None; - unsafe { - self.d3d_device - .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - }; - - let texture = texture.unwrap(); + pub fn as_buffer(&self) -> windows::core::Result> { + let staging_texture = self + .staging_pool + .get_or_create_texture(self.width, self.height)?; - // Copies GPU only texture to CPU-mappable texture unsafe { - self.d3d_context.CopyResource(&texture, &self.texture); + self.d3d_context + .CopyResource(&staging_texture, &self.texture); }; let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { self.d3d_context.Map( - &texture, + &staging_texture, 0, - D3D11_MAP_READ_WRITE, + D3D11_MAP_READ, 0, Some(&mut mapped_resource), )?; }; let data = unsafe { - std::slice::from_raw_parts_mut( + std::slice::from_raw_parts( mapped_resource.pData.cast(), (self.height * mapped_resource.RowPitch) as usize, ) @@ -501,21 +571,31 @@ impl Frame { height: self.height, stride: mapped_resource.RowPitch, pixel_format: self.pixel_format, - resource: mapped_resource, + staging_texture, + d3d_context: self.d3d_context.clone(), }) } } pub struct FrameBuffer<'a> { - data: &'a mut [u8], + data: &'a [u8], width: u32, height: u32, stride: u32, - resource: D3D11_MAPPED_SUBRESOURCE, pixel_format: PixelFormat, + staging_texture: ID3D11Texture2D, + d3d_context: ID3D11DeviceContext, } -impl<'a> FrameBuffer<'a> { +impl Drop for FrameBuffer<'_> { + fn drop(&mut self) { + unsafe { + self.d3d_context.Unmap(&self.staging_texture, 0); + } + } +} + +impl FrameBuffer<'_> { pub fn width(&self) -> u32 { self.width } @@ -532,10 +612,6 @@ impl<'a> FrameBuffer<'a> { self.data } - pub fn inner(&self) -> &D3D11_MAPPED_SUBRESOURCE { - &self.resource - } - pub fn pixel_format(&self) -> PixelFormat { self.pixel_format } diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 0aec041f02..047e5d4e35 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -3,6 +3,33 @@ use scap_direct3d::PixelFormat; pub type AsFFmpegError = windows::core::Error; +#[inline] +fn copy_frame_data( + src_bytes: &[u8], + src_stride: usize, + dest_bytes: &mut [u8], + dest_stride: usize, + row_length: usize, + height: usize, +) { + if src_stride == row_length && dest_stride == row_length { + let total_bytes = row_length * height; + unsafe { + std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), dest_bytes.as_mut_ptr(), total_bytes); + } + } else { + for i in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + src_bytes.as_ptr().add(i * src_stride), + dest_bytes.as_mut_ptr().add(i * dest_stride), + row_length, + ); + } + } + } +} + impl super::AsFFmpeg for scap_direct3d::Frame { fn as_ffmpeg(&self) -> Result { let buffer = self.as_buffer()?; @@ -12,6 +39,7 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let src_bytes = buffer.data(); let src_stride = buffer.stride() as usize; + let row_length = width * 4; match self.pixel_format() { PixelFormat::R8G8B8A8Unorm => { @@ -24,14 +52,14 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); - let row_length = width * 4; - - for i in 0..height { - let src_row = &src_bytes[i * src_stride..i * src_stride + row_length]; - let dest_row = &mut dest_bytes[i * dest_stride..i * dest_stride + row_length]; - - dest_row.copy_from_slice(src_row); - } + copy_frame_data( + src_bytes, + src_stride, + dest_bytes, + dest_stride, + row_length, + height, + ); Ok(ff_frame) } @@ -45,14 +73,14 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); - let row_length = width * 4; - - for i in 0..height { - let src_row = &src_bytes[i * src_stride..i * src_stride + row_length]; - let dest_row = &mut dest_bytes[i * dest_stride..i * dest_stride + row_length]; - - dest_row.copy_from_slice(src_row); - } + copy_frame_data( + src_bytes, + src_stride, + dest_bytes, + dest_stride, + row_length, + height, + ); Ok(ff_frame) } From bc6850f98fd846c39093db8dbecee45920f11abe Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:50:06 +0000 Subject: [PATCH 02/26] Adjust queue and frame pool sizes based on frame rate --- crates/recording/src/output_pipeline/win.rs | 4 +++- crates/recording/src/output_pipeline/win_segmented.rs | 4 +++- crates/recording/src/sources/screen_capture/windows.rs | 2 ++ crates/scap-direct3d/src/lib.rs | 8 +++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index a9c5a756ec..5b635396d6 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -113,7 +113,9 @@ impl Muxer for WindowsMuxer { let output_size = config.output_size.unwrap_or(input_size); let fragmented = config.fragmented; let frag_duration_us = config.frag_duration_us; - let (video_tx, video_rx) = sync_channel::>(8); + let queue_depth = ((config.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); + let (video_tx, video_rx) = + sync_channel::>(queue_depth); let mut output = ffmpeg::format::output(&output_path)?; diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs index bdb8393041..7976712d6e 100644 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ b/crates/recording/src/output_pipeline/win_segmented.rs @@ -354,7 +354,9 @@ impl WindowsSegmentedMuxer { }; let output_size = self.output_size.unwrap_or(input_size); - let (video_tx, video_rx) = sync_channel::>(8); + let queue_depth = ((self.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); + let (video_tx, video_rx) = + sync_channel::>(queue_depth); let (ready_tx, ready_rx) = sync_channel::>(1); let output = ffmpeg::format::output(&segment_path)?; let output = Arc::new(Mutex::new(output)); diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index a62d59e3ce..a73932037b 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -99,6 +99,8 @@ impl ScreenCaptureConfig { Some(Duration::from_secs_f64(1.0 / self.config.fps as f64)); } + settings.fps = Some(self.config.fps); + // Store the display ID instead of GraphicsCaptureItem to avoid COM threading issues // The GraphicsCaptureItem will be created on the capture thread Ok(( diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index c7f3bcd323..77ebf61408 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -164,6 +164,7 @@ pub struct Settings { pub min_update_interval: Option, pub pixel_format: PixelFormat, pub crop: Option, + pub fps: Option, } impl Settings { @@ -298,10 +299,15 @@ impl Capturer { })() .map_err(NewCapturerError::Direct3DDevice)?; + let frame_pool_size = settings + .fps + .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(2, 4)) + .unwrap_or(2); + let frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( &direct3d_device, settings.pixel_format.as_directx(), - 1, + frame_pool_size, item.Size().map_err(NewCapturerError::ItemSize)?, ) .map_err(NewCapturerError::FramePool)?; From 7d9dcaee18ba05e5b9938177b23f7e9058ca4aa2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:55:35 +0000 Subject: [PATCH 03/26] Refactor camera-mediafoundation: use parking_lot and improve logging --- Cargo.lock | 1 + crates/camera-mediafoundation/Cargo.toml | 1 + crates/camera-mediafoundation/src/lib.rs | 39 +++++++++--------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97bceb9bfa..48c6d25ef7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1122,6 +1122,7 @@ version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", "inquire", + "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 0bee6fe27d..36ee6fdd9d 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] tracing.workspace = true thiserror.workspace = true +parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index e4335d9634..5e7610ac9a 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -2,6 +2,7 @@ #![allow(non_snake_case)] use cap_mediafoundation_utils::*; +use parking_lot::Mutex; use std::{ ffi::OsString, fmt::Display, @@ -9,10 +10,7 @@ use std::{ ops::{Deref, DerefMut}, os::windows::ffi::OsStringExt, slice::from_raw_parts, - sync::{ - Mutex, - mpsc::{Receiver, Sender, channel}, - }, + sync::mpsc::{Receiver, Sender, channel}, time::Duration, }; use tracing::error; @@ -227,7 +225,7 @@ impl Device { .SetUINT32(&MF_CAPTURE_ENGINE_USE_VIDEO_DEVICE_ONLY, 1) .map_err(StartCapturingError::ConfigureEngine)?; - println!("Initializing engine..."); + tracing::debug!("Initializing Media Foundation capture engine"); engine .Initialize( @@ -244,7 +242,7 @@ impl Device { )); }; - println!("Engine initialized."); + tracing::debug!("Media Foundation capture engine initialized"); let source = engine .GetSource() @@ -345,8 +343,11 @@ fn retry_on_invalid_request( ) -> windows_core::Result { let mut retry_count = 0; - const MAX_RETRIES: u32 = 100; - const RETRY_DELAY: Duration = Duration::from_millis(50); + const MAX_RETRIES: u32 = 50; + const INITIAL_DELAY_MS: u64 = 1; + const MAX_DELAY_MS: u64 = 50; + + let mut current_delay_ms = INITIAL_DELAY_MS; loop { match cb() { @@ -356,7 +357,8 @@ fn retry_on_invalid_request( return Err(e); } retry_count += 1; - std::thread::sleep(RETRY_DELAY); + std::thread::sleep(Duration::from_millis(current_delay_ms)); + current_delay_ms = (current_delay_ms * 2).min(MAX_DELAY_MS); } Err(e) => return Err(e), } @@ -467,18 +469,9 @@ pub struct VideoSample(IMFSample); impl VideoSample { pub fn bytes(&self) -> windows_core::Result> { unsafe { - let bytes = self.0.GetTotalLength()?; - let mut out = Vec::with_capacity(bytes as usize); - - let buffer_count = self.0.GetBufferCount()?; - for buffer_i in 0..buffer_count { - let buffer = self.0.GetBufferByIndex(buffer_i)?; - - let bytes = buffer.lock()?; - out.extend(&*bytes); - } - - Ok(out) + let buffer = self.0.ConvertToContiguousBuffer()?; + let locked = buffer.lock()?; + Ok(locked.to_vec()) } } } @@ -564,9 +557,7 @@ impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { return Ok(()); }; - let Ok(mut callback) = self.sample_callback.lock() else { - return Ok(()); - }; + let mut callback = self.sample_callback.lock(); let sample_time = unsafe { sample.GetSampleTime() }?; From c0d7ba1a20b885a085d212fa5e0bc4e3969e1d6e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 14:12:09 +0000 Subject: [PATCH 04/26] Add HEVC encoder support to FFmpeg and MediaFoundation --- Cargo.lock | 1 + crates/enc-ffmpeg/Cargo.toml | 3 + crates/enc-ffmpeg/src/video/h264.rs | 65 ++- crates/enc-ffmpeg/src/video/hevc.rs | 507 +++++++++++++++++++ crates/enc-ffmpeg/src/video/mod.rs | 1 + crates/enc-mediafoundation/src/video/h264.rs | 3 +- crates/enc-mediafoundation/src/video/hevc.rs | 450 ++++++++++++++++ crates/enc-mediafoundation/src/video/mod.rs | 2 + crates/frame-converter/Cargo.toml | 1 + crates/frame-converter/src/d3d11.rs | 54 +- 10 files changed, 1069 insertions(+), 18 deletions(-) create mode 100644 crates/enc-ffmpeg/src/video/hevc.rs create mode 100644 crates/enc-mediafoundation/src/video/hevc.rs diff --git a/Cargo.lock b/Cargo.lock index 48c6d25ef7..10649ea5f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,6 +1328,7 @@ dependencies = [ name = "cap-enc-ffmpeg" version = "0.1.0" dependencies = [ + "cap-frame-converter", "cap-media-info", "ffmpeg-next", "serde", diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.toml index cf0244afac..0c010bceee 100644 --- a/crates/enc-ffmpeg/Cargo.toml +++ b/crates/enc-ffmpeg/Cargo.toml @@ -13,5 +13,8 @@ thiserror.workspace = true tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } +[target.'cfg(target_os = "windows")'.dependencies] +cap-frame-converter = { path = "../frame-converter" } + [lints] workspace = true diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 8f8614da4d..fa440c6f93 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -382,6 +382,46 @@ impl H264Encoder { } } +fn get_encoder_priority() -> &'static [&'static str] { + #[cfg(target_os = "macos")] + { + &[ + "h264_videotoolbox", + "h264_qsv", + "h264_nvenc", + "h264_amf", + "h264_mf", + "libx264", + ] + } + + #[cfg(target_os = "windows")] + { + use cap_frame_converter::{GpuVendor, detect_primary_gpu}; + + static ENCODER_PRIORITY_NVIDIA: &[&str] = + &["h264_nvenc", "h264_mf", "h264_qsv", "h264_amf", "libx264"]; + static ENCODER_PRIORITY_AMD: &[&str] = + &["h264_amf", "h264_mf", "h264_nvenc", "h264_qsv", "libx264"]; + static ENCODER_PRIORITY_INTEL: &[&str] = + &["h264_qsv", "h264_mf", "h264_nvenc", "h264_amf", "libx264"]; + static ENCODER_PRIORITY_DEFAULT: &[&str] = + &["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"]; + + match detect_primary_gpu().map(|info| info.vendor) { + Some(GpuVendor::Nvidia) => ENCODER_PRIORITY_NVIDIA, + Some(GpuVendor::Amd) => ENCODER_PRIORITY_AMD, + Some(GpuVendor::Intel) => ENCODER_PRIORITY_INTEL, + _ => ENCODER_PRIORITY_DEFAULT, + } + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + &["libx264"] + } +} + fn get_codec_and_options( config: &VideoInfo, preset: H264Preset, @@ -395,18 +435,7 @@ fn get_codec_and_options( .max(1.0) as i32; let keyframe_interval_str = keyframe_interval.to_string(); - let encoder_priority: &[&str] = if cfg!(target_os = "macos") { - &[ - "h264_videotoolbox", - "h264_qsv", - "h264_nvenc", - "h264_amf", - "h264_mf", - "libx264", - ] - } else { - &["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"] - }; + let encoder_priority = get_encoder_priority(); let mut encoders = Vec::new(); @@ -422,15 +451,21 @@ fn get_codec_and_options( options.set("realtime", "true"); } "h264_nvenc" => { - options.set("preset", "fast"); + options.set("preset", "p4"); + options.set("tune", "ll"); + options.set("rc", "vbr"); + options.set("spatial-aq", "1"); + options.set("temporal-aq", "1"); options.set("g", &keyframe_interval_str); } "h264_qsv" => { - options.set("preset", "fast"); + options.set("preset", "faster"); + options.set("look_ahead", "1"); options.set("g", &keyframe_interval_str); } "h264_amf" => { - options.set("quality", "speed"); + options.set("quality", "balanced"); + options.set("rc", "vbr_latency"); options.set("g", &keyframe_interval_str); } "h264_mf" => { diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rs new file mode 100644 index 0000000000..f9548959bc --- /dev/null +++ b/crates/enc-ffmpeg/src/video/hevc.rs @@ -0,0 +1,507 @@ +use std::{thread, time::Duration}; + +use cap_media_info::{Pixel, VideoInfo}; +use ffmpeg::{ + Dictionary, + codec::{codec::Codec, context, encoder}, + format::{self}, + frame, + threading::Config, +}; +use tracing::{debug, error, trace}; + +use crate::base::EncoderBase; + +fn is_420(format: ffmpeg::format::Pixel) -> bool { + format + .descriptor() + .map(|desc| desc.log2_chroma_w() == 1 && desc.log2_chroma_h() == 1) + .unwrap_or(false) +} + +pub struct HevcEncoderBuilder { + bpp: f32, + input_config: VideoInfo, + preset: HevcPreset, + output_size: Option<(u32, u32)>, + external_conversion: bool, +} + +#[derive(Clone, Copy)] +pub enum HevcPreset { + Slow, + Medium, + Ultrafast, +} + +#[derive(thiserror::Error, Debug)] +pub enum HevcEncoderError { + #[error("{0:?}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Codec not found")] + CodecNotFound, + #[error("Pixel format {0:?} not supported")] + PixFmtNotSupported(Pixel), + #[error("Invalid output dimensions {width}x{height}; expected non-zero even width and height")] + InvalidOutputDimensions { width: u32, height: u32 }, +} + +impl HevcEncoderBuilder { + pub const QUALITY_BPP: f32 = 0.2; + + pub fn new(input_config: VideoInfo) -> Self { + Self { + input_config, + bpp: Self::QUALITY_BPP, + preset: HevcPreset::Ultrafast, + output_size: None, + external_conversion: false, + } + } + + pub fn with_preset(mut self, preset: HevcPreset) -> Self { + self.preset = preset; + self + } + + pub fn with_bpp(mut self, bpp: f32) -> Self { + self.bpp = bpp; + self + } + + pub fn with_output_size(mut self, width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Err(HevcEncoderError::InvalidOutputDimensions { width, height }); + } + + self.output_size = Some((width, height)); + Ok(self) + } + + pub fn with_external_conversion(mut self) -> Self { + self.external_conversion = true; + self + } + + pub fn build( + self, + output: &mut format::context::Output, + ) -> Result { + let input_config = self.input_config; + let (output_width, output_height) = self + .output_size + .unwrap_or((input_config.width, input_config.height)); + + if output_width == 0 || output_height == 0 { + return Err(HevcEncoderError::InvalidOutputDimensions { + width: output_width, + height: output_height, + }); + } + + let candidates = get_codec_and_options(&input_config, self.preset); + if candidates.is_empty() { + return Err(HevcEncoderError::CodecNotFound); + } + + let mut last_error = None; + + for (codec, encoder_options) in candidates { + let codec_name = codec.name().to_string(); + + match Self::build_with_codec( + codec, + encoder_options, + &input_config, + output, + output_width, + output_height, + self.bpp, + self.external_conversion, + ) { + Ok(encoder) => { + debug!("Using HEVC encoder {}", codec_name); + return Ok(encoder); + } + Err(err) => { + debug!("HEVC encoder {} init failed: {:?}", codec_name, err); + last_error = Some(err); + } + } + } + + Err(last_error.unwrap_or(HevcEncoderError::CodecNotFound)) + } + + #[allow(clippy::too_many_arguments)] + fn build_with_codec( + codec: Codec, + encoder_options: Dictionary<'static>, + input_config: &VideoInfo, + output: &mut format::context::Output, + output_width: u32, + output_height: u32, + bpp: f32, + external_conversion: bool, + ) -> Result { + let encoder_supports_input_format = codec + .video() + .ok() + .and_then(|codec_video| codec_video.formats()) + .is_some_and(|mut formats| formats.any(|f| f == input_config.pixel_format)); + + let mut needs_pixel_conversion = false; + + let output_format = if encoder_supports_input_format { + input_config.pixel_format + } else { + needs_pixel_conversion = true; + let format = ffmpeg::format::Pixel::NV12; + if !external_conversion { + debug!( + "Converting from {:?} to {:?} for HEVC encoding", + input_config.pixel_format, format + ); + } + format + }; + + if is_420(output_format) + && (!output_width.is_multiple_of(2) || !output_height.is_multiple_of(2)) + { + return Err(HevcEncoderError::InvalidOutputDimensions { + width: output_width, + height: output_height, + }); + } + + let needs_scaling = + output_width != input_config.width || output_height != input_config.height; + + if needs_scaling && !external_conversion { + debug!( + "Scaling video frames for HEVC encoding from {}x{} to {}x{}", + input_config.width, input_config.height, output_width, output_height + ); + } + + let converter = if external_conversion { + debug!( + "External conversion enabled, skipping internal converter. Expected input: {:?} {}x{}", + output_format, output_width, output_height + ); + None + } else if needs_pixel_conversion || needs_scaling { + let flags = if needs_scaling { + ffmpeg::software::scaling::flag::Flags::BICUBIC + } else { + ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR + }; + + match ffmpeg::software::scaling::Context::get( + input_config.pixel_format, + input_config.width, + input_config.height, + output_format, + output_width, + output_height, + flags, + ) { + Ok(context) => Some(context), + Err(e) => { + if needs_pixel_conversion { + error!( + "Failed to create converter from {:?} to {:?}: {:?}", + input_config.pixel_format, output_format, e + ); + return Err(HevcEncoderError::PixFmtNotSupported( + input_config.pixel_format, + )); + } + + return Err(HevcEncoderError::FFmpeg(e)); + } + } + } else { + None + }; + + let mut encoder_ctx = context::Context::new_with_codec(codec); + + let thread_count = thread::available_parallelism() + .map(|v| v.get()) + .unwrap_or(1); + encoder_ctx.set_threading(Config::count(thread_count)); + let mut encoder = encoder_ctx.encoder().video()?; + + encoder.set_width(output_width); + encoder.set_height(output_height); + encoder.set_format(output_format); + encoder.set_time_base(input_config.time_base); + encoder.set_frame_rate(Some(input_config.frame_rate)); + + let bitrate = get_bitrate( + output_width, + output_height, + input_config.frame_rate.0 as f32 / input_config.frame_rate.1 as f32, + bpp, + ); + + encoder.set_bit_rate(bitrate); + encoder.set_max_bit_rate(bitrate); + + let encoder = encoder.open_with(encoder_options)?; + + let mut output_stream = output.add_stream(codec)?; + let stream_index = output_stream.index(); + output_stream.set_time_base((1, HevcEncoder::TIME_BASE)); + output_stream.set_rate(input_config.frame_rate); + output_stream.set_parameters(&encoder); + + Ok(HevcEncoder { + base: EncoderBase::new(stream_index), + encoder, + converter, + output_format, + output_width, + output_height, + input_format: input_config.pixel_format, + input_width: input_config.width, + input_height: input_config.height, + }) + } +} + +pub struct HevcEncoder { + base: EncoderBase, + encoder: encoder::Video, + converter: Option, + output_format: format::Pixel, + output_width: u32, + output_height: u32, + input_format: format::Pixel, + input_width: u32, + input_height: u32, +} + +pub struct ConversionRequirements { + pub input_format: format::Pixel, + pub input_width: u32, + pub input_height: u32, + pub output_format: format::Pixel, + pub output_width: u32, + pub output_height: u32, + pub needs_conversion: bool, +} + +#[derive(thiserror::Error, Debug)] +pub enum QueueFrameError { + #[error("Converter: {0}")] + Converter(ffmpeg::Error), + #[error("Encode: {0}")] + Encode(ffmpeg::Error), +} + +impl HevcEncoder { + const TIME_BASE: i32 = 90000; + + pub fn builder(input_config: VideoInfo) -> HevcEncoderBuilder { + HevcEncoderBuilder::new(input_config) + } + + pub fn conversion_requirements(&self) -> ConversionRequirements { + let needs_conversion = self.input_format != self.output_format + || self.input_width != self.output_width + || self.input_height != self.output_height; + ConversionRequirements { + input_format: self.input_format, + input_width: self.input_width, + input_height: self.input_height, + output_format: self.output_format, + output_width: self.output_width, + output_height: self.output_height, + needs_conversion, + } + } + + pub fn queue_frame( + &mut self, + mut frame: frame::Video, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + self.base + .update_pts(&mut frame, timestamp, &mut self.encoder); + + if let Some(converter) = &mut self.converter { + let pts = frame.pts(); + let mut converted = + frame::Video::new(self.output_format, self.output_width, self.output_height); + converter + .run(&frame, &mut converted) + .map_err(QueueFrameError::Converter)?; + converted.set_pts(pts); + frame = converted; + } + + self.base + .send_frame(&frame, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + + pub fn queue_preconverted_frame( + &mut self, + mut frame: frame::Video, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + trace!( + "Encoding pre-converted frame: format={:?}, size={}x{}, expected={:?} {}x{}", + frame.format(), + frame.width(), + frame.height(), + self.output_format, + self.output_width, + self.output_height + ); + + self.base + .update_pts(&mut frame, timestamp, &mut self.encoder); + + self.base + .send_frame(&frame, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + + pub fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { + self.base.process_eof(output, &mut self.encoder) + } +} + +fn get_encoder_priority() -> &'static [&'static str] { + #[cfg(target_os = "macos")] + { + &[ + "hevc_videotoolbox", + "hevc_qsv", + "hevc_nvenc", + "hevc_amf", + "hevc_mf", + "libx265", + ] + } + + #[cfg(target_os = "windows")] + { + use cap_frame_converter::{GpuVendor, detect_primary_gpu}; + + static ENCODER_PRIORITY_NVIDIA: &[&str] = + &["hevc_nvenc", "hevc_mf", "hevc_qsv", "hevc_amf", "libx265"]; + static ENCODER_PRIORITY_AMD: &[&str] = + &["hevc_amf", "hevc_mf", "hevc_nvenc", "hevc_qsv", "libx265"]; + static ENCODER_PRIORITY_INTEL: &[&str] = + &["hevc_qsv", "hevc_mf", "hevc_nvenc", "hevc_amf", "libx265"]; + static ENCODER_PRIORITY_DEFAULT: &[&str] = + &["hevc_nvenc", "hevc_qsv", "hevc_amf", "hevc_mf", "libx265"]; + + match detect_primary_gpu().map(|info| info.vendor) { + Some(GpuVendor::Nvidia) => ENCODER_PRIORITY_NVIDIA, + Some(GpuVendor::Amd) => ENCODER_PRIORITY_AMD, + Some(GpuVendor::Intel) => ENCODER_PRIORITY_INTEL, + _ => ENCODER_PRIORITY_DEFAULT, + } + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + &["libx265"] + } +} + +fn get_codec_and_options( + config: &VideoInfo, + preset: HevcPreset, +) -> Vec<(Codec, Dictionary<'static>)> { + let keyframe_interval_secs = 2; + let denominator = config.frame_rate.denominator(); + let frames_per_sec = config.frame_rate.numerator() as f64 + / if denominator == 0 { 1 } else { denominator } as f64; + let keyframe_interval = (keyframe_interval_secs as f64 * frames_per_sec) + .round() + .max(1.0) as i32; + let keyframe_interval_str = keyframe_interval.to_string(); + + let encoder_priority = get_encoder_priority(); + + let mut encoders = Vec::new(); + + for encoder_name in encoder_priority { + let Some(codec) = encoder::find_by_name(encoder_name) else { + continue; + }; + + let mut options = Dictionary::new(); + + match *encoder_name { + "hevc_videotoolbox" => { + options.set("realtime", "true"); + } + "hevc_nvenc" => { + options.set("preset", "p4"); + options.set("tune", "ll"); + options.set("rc", "vbr"); + options.set("spatial-aq", "1"); + options.set("temporal-aq", "1"); + options.set("tier", "main"); + options.set("g", &keyframe_interval_str); + } + "hevc_qsv" => { + options.set("preset", "faster"); + options.set("look_ahead", "1"); + options.set("g", &keyframe_interval_str); + } + "hevc_amf" => { + options.set("quality", "balanced"); + options.set("rc", "vbr_latency"); + options.set("g", &keyframe_interval_str); + } + "hevc_mf" => { + options.set("hw_encoding", "true"); + options.set("scenario", "4"); + options.set("quality", "1"); + options.set("g", &keyframe_interval_str); + } + "libx265" => { + options.set( + "preset", + match preset { + HevcPreset::Slow => "slow", + HevcPreset::Medium => "medium", + HevcPreset::Ultrafast => "ultrafast", + }, + ); + if let HevcPreset::Ultrafast = preset { + options.set("tune", "zerolatency"); + } + options.set("g", &keyframe_interval_str); + } + _ => {} + } + + encoders.push((codec, options)); + } + + encoders +} + +fn get_bitrate(width: u32, height: u32, frame_rate: f32, bpp: f32) -> usize { + let frame_rate_multiplier = ((frame_rate as f64 - 30.0).max(0.0) * 0.6) + 30.0; + let area = (width as f64) * (height as f64); + let pixels_per_second = area * frame_rate_multiplier; + + (pixels_per_second * bpp as f64) as usize +} diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs index f61e796943..3429498fe7 100644 --- a/crates/enc-ffmpeg/src/video/mod.rs +++ b/crates/enc-ffmpeg/src/video/mod.rs @@ -1 +1,2 @@ pub mod h264; +pub mod hevc; diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 4fae2c1890..dd371989c2 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -476,5 +476,6 @@ impl H264Encoder { } fn calculate_bitrate(width: u32, height: u32, fps: u32, multiplier: f32) -> u32 { - ((width * height * ((fps - 30) / 2 + 30)) as f32 * multiplier) as u32 + let frame_rate_factor = (fps as f32 - 30.0).max(0.0) / 2.0 + 30.0; + (width as f32 * height as f32 * frame_rate_factor * multiplier) as u32 } diff --git a/crates/enc-mediafoundation/src/video/hevc.rs b/crates/enc-mediafoundation/src/video/hevc.rs new file mode 100644 index 0000000000..2b4a72ec08 --- /dev/null +++ b/crates/enc-mediafoundation/src/video/hevc.rs @@ -0,0 +1,450 @@ +use crate::{ + media::{MFSetAttributeRatio, MFSetAttributeSize}, + mft::EncoderDevice, + video::{NewVideoProcessorError, VideoProcessor}, +}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use windows::{ + Foundation::TimeSpan, + Graphics::SizeInt32, + Win32::{ + Foundation::E_NOTIMPL, + Graphics::{ + Direct3D11::{ID3D11Device, ID3D11Texture2D}, + Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12}, + }, + Media::MediaFoundation::{ + self, IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, + IMFSample, IMFTransform, MF_E_INVALIDMEDIATYPE, MF_E_NO_MORE_TYPES, + MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, + MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, + MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, + MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, + MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, + MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH, + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, + MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, + MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, + MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoInterlace_Progressive, + }, + }, + core::{Error, Interface}, +}; + +const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; + +pub struct HevcEncoder { + _d3d_device: ID3D11Device, + _media_device_manager: IMFDXGIDeviceManager, + _device_manager_reset_token: u32, + + video_processor: VideoProcessor, + + transform: IMFTransform, + event_generator: IMFMediaEventGenerator, + input_stream_id: u32, + output_stream_id: u32, + output_type: IMFMediaType, + bitrate: u32, +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum NewHevcEncoderError { + #[error("NoVideoEncoderDevice")] + NoVideoEncoderDevice, + #[error("EncoderTransform: {0}")] + EncoderTransform(windows::core::Error), + #[error("VideoProcessor: {0}")] + VideoProcessor(NewVideoProcessorError), + #[error("DeviceManager: {0}")] + DeviceManager(windows::core::Error), + #[error("EventGenerator: {0}")] + EventGenerator(windows::core::Error), + #[error("ConfigureStreams: {0}")] + ConfigureStreams(windows::core::Error), + #[error("OutputType: {0}")] + OutputType(windows::core::Error), + #[error("InputType: {0}")] + InputType(windows::core::Error), +} + +unsafe impl Send for HevcEncoder {} + +impl HevcEncoder { + #[allow(clippy::too_many_arguments)] + fn new_with_scaled_output_with_flags( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + flags: MFT_ENUM_FLAG, + enable_hardware_transforms: bool, + ) -> Result { + let bitrate = calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ); + + let transform = + EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_HEVC, flags) + .map_err(|_| NewHevcEncoderError::NoVideoEncoderDevice)? + .first() + .cloned() + .ok_or(NewHevcEncoderError::NoVideoEncoderDevice)? + .create_transform() + .map_err(NewHevcEncoderError::EncoderTransform)?; + + let video_processor = VideoProcessor::new( + d3d_device.clone(), + format, + input_resolution, + DXGI_FORMAT_NV12, + output_resolution, + frame_rate, + ) + .map_err(NewHevcEncoderError::VideoProcessor)?; + + let mut device_manager_reset_token: u32 = 0; + let media_device_manager = { + let mut media_device_manager = None; + unsafe { + MFCreateDXGIDeviceManager( + &mut device_manager_reset_token, + &mut media_device_manager, + ) + .map_err(NewHevcEncoderError::DeviceManager)? + }; + media_device_manager.expect("Device manager unexpectedly None") + }; + unsafe { + media_device_manager + .ResetDevice(d3d_device, device_manager_reset_token) + .map_err(NewHevcEncoderError::DeviceManager)? + }; + + let event_generator: IMFMediaEventGenerator = transform + .cast() + .map_err(NewHevcEncoderError::EventGenerator)?; + let attributes = unsafe { + transform + .GetAttributes() + .map_err(NewHevcEncoderError::EventGenerator)? + }; + unsafe { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .map_err(NewHevcEncoderError::EventGenerator)?; + attributes + .SetUINT32( + &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + enable_hardware_transforms as u32, + ) + .map_err(NewHevcEncoderError::EventGenerator)?; + }; + + let mut number_of_input_streams = 0; + let mut number_of_output_streams = 0; + unsafe { + transform + .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) + .map_err(NewHevcEncoderError::EventGenerator)? + }; + let (input_stream_ids, output_stream_ids) = { + let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; + let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; + let result = + unsafe { transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) }; + match result { + Ok(_) => {} + Err(error) => { + if error.code() == E_NOTIMPL { + for i in 0..number_of_input_streams { + input_stream_ids[i as usize] = i; + } + for i in 0..number_of_output_streams { + output_stream_ids[i as usize] = i; + } + } else { + return Err(NewHevcEncoderError::ConfigureStreams(error)); + } + } + } + (input_stream_ids, output_stream_ids) + }; + let input_stream_id = input_stream_ids[0]; + let output_stream_id = output_stream_ids[0]; + + unsafe { + let temp = media_device_manager.clone(); + transform + .ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + std::mem::transmute::(temp), + ) + .map_err(NewHevcEncoderError::EncoderTransform)?; + }; + + let output_type = (|| unsafe { + let output_type = MFCreateMediaType()?; + let attributes: IMFAttributes = output_type.cast()?; + output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_HEVC)?; + output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; + output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; + transform.SetOutputType(output_stream_id, &output_type, 0)?; + Ok(output_type) + })() + .map_err(NewHevcEncoderError::OutputType)?; + + let input_type: Option = (|| unsafe { + let mut count = 0; + loop { + let result = transform.GetInputAvailableType(input_stream_id, count); + if let Err(error) = &result + && error.code() == MF_E_NO_MORE_TYPES + { + break Ok(None); + } + + let input_type = result?; + let attributes: IMFAttributes = input_type.cast()?; + input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + let result = transform.SetInputType( + input_stream_id, + &input_type, + MFT_SET_TYPE_TEST_ONLY.0 as u32, + ); + if let Err(error) = &result + && error.code() == MF_E_INVALIDMEDIATYPE + { + count += 1; + continue; + } + result?; + break Ok(Some(input_type)); + } + })() + .map_err(NewHevcEncoderError::InputType)?; + if let Some(input_type) = input_type { + unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } + .map_err(NewHevcEncoderError::InputType)?; + } else { + return Err(NewHevcEncoderError::InputType(Error::new( + MF_E_TRANSFORM_TYPE_NOT_SET, + "No suitable input type found! Try a different set of encoding settings.", + ))); + } + + Ok(Self { + _d3d_device: d3d_device.clone(), + _media_device_manager: media_device_manager, + _device_manager_reset_token: device_manager_reset_token, + + video_processor, + + transform, + event_generator, + input_stream_id, + output_stream_id, + bitrate, + + output_type, + }) + } + + pub fn new_with_scaled_output( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY, + true, + ) + } + + pub fn new_with_scaled_output_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_TRANSCODE_ONLY, + false, + ) + } + + pub fn new( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + + pub fn new_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_software( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + + pub fn bitrate(&self) -> u32 { + self.bitrate + } + + pub fn output_type(&self) -> &IMFMediaType { + &self.output_type + } + + pub fn run( + &mut self, + should_stop: Arc, + mut get_frame: impl FnMut() -> windows::core::Result>, + mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, + ) -> windows::core::Result<()> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; + + let mut consecutive_empty_samples = 0; + let mut should_exit = false; + while !should_exit { + let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; + + let event_type = MF_EVENT_TYPE(event.GetType()? as i32); + match event_type { + MediaFoundation::METransformNeedInput => { + should_exit = true; + if !should_stop.load(Ordering::SeqCst) + && let Some((texture, timestamp)) = get_frame()? + { + self.video_processor.process_texture(&texture)?; + let input_buffer = { + MFCreateDXGISurfaceBuffer( + &ID3D11Texture2D::IID, + self.video_processor.output_texture(), + 0, + false, + )? + }; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + should_exit = false; + } + } + MediaFoundation::METransformHaveOutput => { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + + let mut output_buffers = [output_buffer]; + self.transform + .ProcessOutput(0, &mut output_buffers, &mut status)?; + + if let Some(sample) = output_buffers[0].pSample.take() { + consecutive_empty_samples = 0; + on_sample(sample)?; + } else { + consecutive_empty_samples += 1; + if consecutive_empty_samples > MAX_CONSECUTIVE_EMPTY_SAMPLES { + return Err(windows::core::Error::new( + windows::core::HRESULT(0), + "Too many consecutive empty samples", + )); + } + } + } + _ => { + panic!("Unknown media event type: {}", event_type.0); + } + } + } + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + } + + Ok(()) + } +} + +fn calculate_bitrate(width: u32, height: u32, fps: u32, multiplier: f32) -> u32 { + let frame_rate_factor = (fps as f32 - 30.0).max(0.0) / 2.0 + 30.0; + (width as f32 * height as f32 * frame_rate_factor * multiplier * 0.6) as u32 +} diff --git a/crates/enc-mediafoundation/src/video/mod.rs b/crates/enc-mediafoundation/src/video/mod.rs index 273711713b..b6376900bd 100644 --- a/crates/enc-mediafoundation/src/video/mod.rs +++ b/crates/enc-mediafoundation/src/video/mod.rs @@ -1,5 +1,7 @@ mod h264; +mod hevc; mod video_processor; pub use h264::*; +pub use hevc::*; pub use video_processor::*; diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index a69a81dbba..37cdc886ce 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -24,6 +24,7 @@ windows = { workspace = true, features = [ "Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Media_MediaFoundation", ] } diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 60eaaefd45..767756ca1f 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -4,7 +4,10 @@ use parking_lot::Mutex; use std::{ mem::ManuallyDrop, ptr, - sync::atomic::{AtomicBool, AtomicU64, Ordering}, + sync::{ + OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, }; use windows::{ Win32::{ @@ -25,7 +28,7 @@ use windows::{ }, Dxgi::{ Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_YUY2}, - IDXGIAdapter, IDXGIDevice, + CreateDXGIFactory1, IDXGIAdapter, IDXGIDevice, IDXGIFactory1, }, }, }, @@ -80,6 +83,53 @@ impl GpuInfo { } } +static DETECTED_GPU: OnceLock> = OnceLock::new(); + +pub fn detect_primary_gpu() -> Option<&'static GpuInfo> { + DETECTED_GPU + .get_or_init(|| { + let result = detect_primary_gpu_inner(); + if let Some(ref info) = result { + tracing::debug!( + "Detected primary GPU: {} (Vendor: {}, VendorID: 0x{:04X}, VRAM: {} MB)", + info.description, + info.vendor_name(), + info.vendor_id, + info.dedicated_video_memory / (1024 * 1024) + ); + } else { + tracing::debug!("No GPU detected via DXGI, using default encoder order"); + } + result + }) + .as_ref() +} + +fn detect_primary_gpu_inner() -> Option { + unsafe { + let factory: IDXGIFactory1 = CreateDXGIFactory1().ok()?; + let adapter: IDXGIAdapter = factory.EnumAdapters(0).ok()?; + let desc = adapter.GetDesc().ok()?; + + let description = String::from_utf16_lossy( + &desc + .Description + .iter() + .take_while(|&&c| c != 0) + .copied() + .collect::>(), + ); + + Some(GpuInfo { + vendor: GpuVendor::from_id(desc.VendorId), + vendor_id: desc.VendorId, + device_id: desc.DeviceId, + description, + dedicated_video_memory: desc.DedicatedVideoMemory as u64, + }) + } +} + struct D3D11Resources { #[allow(dead_code)] device: ID3D11Device, From 6942ae95e11fe82b400bed5ef2d9233baa7312ea Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 14:44:29 +0000 Subject: [PATCH 05/26] Add support for shared D3D11 texture handles --- crates/frame-converter/Cargo.toml | 1 + crates/frame-converter/src/d3d11.rs | 150 +++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 15 deletions(-) diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index 37cdc886ce..39b4e7ce78 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -27,5 +27,6 @@ windows = { workspace = true, features = [ "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Media_MediaFoundation", + "Win32_Security", ] } diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 767756ca1f..9f0c44e377 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -11,24 +11,25 @@ use std::{ }; use windows::{ Win32::{ - Foundation::HMODULE, + Foundation::{CloseHandle, HANDLE, HMODULE}, Graphics::{ Direct3D::D3D_DRIVER_TYPE_HARDWARE, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAP_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, - D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, - D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VPIV_DIMENSION_TEXTURE2D, - D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, ID3D11Device, - ID3D11DeviceContext, ID3D11Texture2D, ID3D11VideoContext, ID3D11VideoDevice, - ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, + D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_SDK_VERSION, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, + D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_STREAM, + D3D11_VPIV_DIMENSION_TEXTURE2D, D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, + ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, ID3D11VideoContext, + ID3D11VideoDevice, ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, }, Dxgi::{ Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_YUY2}, - CreateDXGIFactory1, IDXGIAdapter, IDXGIDevice, IDXGIFactory1, + CreateDXGIFactory1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, + IDXGIAdapter, IDXGIDevice, IDXGIFactory1, IDXGIResource1, }, }, }, @@ -140,6 +141,8 @@ struct D3D11Resources { enumerator: ID3D11VideoProcessorEnumerator, input_texture: ID3D11Texture2D, output_texture: ID3D11Texture2D, + input_shared_handle: Option, + output_shared_handle: Option, staging_input: ID3D11Texture2D, staging_output: ID3D11Texture2D, } @@ -287,24 +290,20 @@ impl D3D11Converter { })? }; - let input_texture = create_texture( + let (input_texture, input_shared_handle) = create_shared_texture( &device, config.input_width, config.input_height, input_dxgi, - D3D11_USAGE_DEFAULT, D3D11_BIND_RENDER_TARGET.0 as u32, - 0, )?; - let output_texture = create_texture( + let (output_texture, output_shared_handle) = create_shared_texture( &device, config.output_width, config.output_height, output_dxgi, - D3D11_USAGE_DEFAULT, D3D11_BIND_RENDER_TARGET.0 as u32, - 0, )?; let staging_input = create_texture( @@ -327,6 +326,16 @@ impl D3D11Converter { D3D11_CPU_ACCESS_READ.0 as u32, )?; + if input_shared_handle.is_some() && output_shared_handle.is_some() { + tracing::info!("D3D11 converter created with shared texture handles enabled"); + } else { + tracing::warn!( + "D3D11 converter created without shared handles (input: {}, output: {})", + input_shared_handle.is_some(), + output_shared_handle.is_some() + ); + } + let resources = D3D11Resources { device, context, @@ -336,6 +345,8 @@ impl D3D11Converter { enumerator, input_texture, output_texture, + input_shared_handle, + output_shared_handle, staging_input, staging_output, }; @@ -368,6 +379,23 @@ impl D3D11Converter { pub fn gpu_info(&self) -> &GpuInfo { &self.gpu_info } + + pub fn input_shared_handle(&self) -> Option { + self.resources.lock().input_shared_handle + } + + pub fn output_shared_handle(&self) -> Option { + self.resources.lock().output_shared_handle + } + + pub fn output_texture(&self) -> ID3D11Texture2D { + self.resources.lock().output_texture.clone() + } + + pub fn has_shared_handles(&self) -> bool { + let resources = self.resources.lock(); + resources.input_shared_handle.is_some() && resources.output_shared_handle.is_some() + } } impl FrameConverter for D3D11Converter { @@ -582,6 +610,81 @@ fn create_texture( } } +fn create_shared_texture( + device: &ID3D11Device, + width: u32, + height: u32, + format: DXGI_FORMAT, + bind_flags: u32, +) -> Result<(ID3D11Texture2D, Option), ConvertError> { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: format, + SampleDesc: windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: bind_flags, + CPUAccessFlags: 0, + MiscFlags: D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0 as u32, + }; + + let texture = unsafe { + let mut texture: Option = None; + device + .CreateTexture2D(&desc, None, Some(&mut texture)) + .map_err(|e| { + ConvertError::HardwareUnavailable(format!( + "CreateTexture2D with shared handle failed: {e:?}" + )) + })?; + texture.ok_or_else(|| { + ConvertError::HardwareUnavailable("CreateTexture2D returned null".to_string()) + })? + }; + + let shared_handle = extract_shared_handle(&texture); + + Ok((texture, shared_handle)) +} + +fn extract_shared_handle(texture: &ID3D11Texture2D) -> Option { + unsafe { + let dxgi_resource: Result = texture.cast(); + match dxgi_resource { + Ok(resource) => { + let result = resource.CreateSharedHandle( + None, + DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0, + None, + ); + match result { + Ok(handle) if !handle.is_invalid() => { + tracing::debug!("Created shared handle for texture: {:?}", handle); + Some(handle) + } + Ok(_) => { + tracing::warn!("CreateSharedHandle returned invalid handle"); + None + } + Err(e) => { + tracing::warn!("CreateSharedHandle failed: {e:?}"); + None + } + } + } + Err(e) => { + tracing::warn!("Failed to cast texture to IDXGIResource1: {e:?}"); + None + } + } + } +} + unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: usize) { let height = frame.height() as usize; let format = frame.format(); @@ -669,3 +772,20 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr unsafe impl Send for D3D11Converter {} unsafe impl Sync for D3D11Converter {} + +impl Drop for D3D11Resources { + fn drop(&mut self) { + unsafe { + if let Some(handle) = self.input_shared_handle.take() { + if !handle.is_invalid() { + let _ = CloseHandle(handle); + } + } + if let Some(handle) = self.output_shared_handle.take() { + if !handle.is_invalid() { + let _ = CloseHandle(handle); + } + } + } + } +} From 4a97110b5a2686508302db6740b306784847f62e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 15:12:47 +0000 Subject: [PATCH 06/26] Add encoder preferences and software fallback for camera muxers --- Cargo.lock | 1 + crates/recording/src/output_pipeline/win.rs | 379 ++++++++++++------ .../output_pipeline/win_segmented_camera.rs | 251 ++++++++---- crates/recording/src/studio_recording.rs | 12 +- crates/scap-direct3d/Cargo.toml | 1 + crates/scap-direct3d/src/lib.rs | 105 +++-- 6 files changed, 525 insertions(+), 224 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10649ea5f7..1cbb176ce2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7726,6 +7726,7 @@ dependencies = [ "scap-ffmpeg", "scap-targets", "thiserror 1.0.69", + "tracing", "windows 0.60.0", "windows-numerics 0.2.0", "workspace-hack", diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 5b635396d6..b332813b8e 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -435,6 +435,7 @@ pub struct WindowsCameraMuxerConfig { pub output_height: Option, pub fragmented: bool, pub frag_duration_us: i64, + pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, } impl Default for WindowsCameraMuxerConfig { @@ -443,6 +444,7 @@ impl Default for WindowsCameraMuxerConfig { output_height: None, fragmented: false, frag_duration_us: 2_000_000, + encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), } } } @@ -501,6 +503,7 @@ impl Muxer for WindowsCameraMuxer { { let output = output.clone(); + let encoder_preferences = config.encoder_preferences; tasks.spawn_thread("windows-camera-encoder", move || { cap_mediafoundation_utils::thread_init(); @@ -527,132 +530,230 @@ impl Muxer for WindowsCameraMuxer { let input_format = first_frame.0.dxgi_format(); - let encoder_result = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ); - - let (mut encoder, mut muxer) = match encoder_result { - Ok(encoder) => { - let muxer = { - let mut output_guard = match output.lock() { - Ok(guard) => guard, - Err(poisoned) => { - let msg = format!("Failed to lock output mutex: {poisoned}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); - } - }; + let encoder = (|| { + let fallback = |reason: Option| { + encoder_preferences.force_software_only(); + if let Some(reason) = reason.as_ref() { + error!( + "Falling back to software H264 encoder for camera: {reason}" + ); + } else { + info!("Using software H264 encoder for camera"); + } - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented, - frag_duration_us, - }, - ) + let mut output_guard = match output.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return Err(anyhow!( + "CameraSoftwareEncoder: failed to lock output mutex: {}", + poisoned + )); + } }; - match muxer { - Ok(muxer) => (encoder, muxer), - Err(err) => { - let msg = format!("Failed to create muxer: {err}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); + cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) + .with_output_size(output_width, output_height) + .and_then(|builder| builder.build(&mut output_guard)) + .map(either::Right) + .map_err(|e| anyhow!("CameraSoftwareEncoder/{e}")) + }; + + if encoder_preferences.should_force_software() { + return fallback(None); + } + + match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + &d3d_device, + input_format, + input_size, + output_size, + frame_rate, + bitrate_multiplier, + ) { + Ok(encoder) => { + let muxer = { + let mut output_guard = match output.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return fallback(Some(format!( + "Failed to lock output mutex: {poisoned}" + ))); + } + }; + + cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output_guard, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: output_width, + height: output_height, + fps: frame_rate, + bitrate: encoder.bitrate(), + fragmented, + frag_duration_us, + }, + ) + }; + + match muxer { + Ok(muxer) => Ok(either::Left((encoder, muxer))), + Err(err) => fallback(Some(err.to_string())), } } + Err(err) => fallback(Some(err.to_string())), } - Err(err) => { - let msg = format!("Failed to create H264 encoder: {err}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); + })(); + + let encoder = match encoder { + Ok(encoder) => { + if ready_tx.send(Ok(())).is_err() { + error!("Failed to send ready signal - receiver dropped"); + return Ok(()); + } + encoder + } + Err(e) => { + error!("Camera encoder setup failed: {:#}", e); + let _ = ready_tx.send(Err(anyhow!("{e}"))); + return Err(anyhow!("{e}")); } }; - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } + match encoder { + either::Left((mut encoder, mut muxer)) => { + info!( + "Windows camera encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", + input_format, + input_size.Width, + input_size.Height, + output_size.Width, + output_size.Height, + frame_rate + ); - info!( - "Windows camera encoder started: {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut frame_count = 0u64; - - let mut process_frame = |frame: NativeCameraFrame, - timestamp: Duration| - -> windows::core::Result< - Option<( - windows::Win32::Graphics::Direct3D11::ID3D11Texture2D, - TimeSpan, - )>, - > { - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; + let mut first_timestamp: Option = None; + let mut frame_count = 0u64; + + let mut process_frame = |frame: NativeCameraFrame, + timestamp: Duration| + -> windows::core::Result< + Option<( + windows::Win32::Graphics::Direct3D11::ID3D11Texture2D, + TimeSpan, + )>, + > { + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }; + let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; + Ok(Some((texture, duration_to_timespan(relative)))) + }; - if let Ok(Some((texture, frame_time))) = process_frame(first_frame.0, first_frame.1) - { - encoder - .run( - Arc::new(AtomicBool::default()), - || { - if frame_count > 0 { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available"); - return Ok(None); - }; - frame_count += 1; - if frame_count.is_multiple_of(30) { - debug!( - "Windows camera encoder: processed {} frames", - frame_count - ); - } - return process_frame(frame, timestamp); - } + if let Ok(Some((texture, frame_time))) = + process_frame(first_frame.0, first_frame.1) + { + encoder + .run( + Arc::new(AtomicBool::default()), + || { + if frame_count > 0 { + let Ok(Some((frame, timestamp))) = video_rx.recv() + else { + trace!("No more camera frames available"); + return Ok(None); + }; + frame_count += 1; + if frame_count.is_multiple_of(30) { + debug!( + "Windows camera encoder: processed {} frames", + frame_count + ); + } + return process_frame(frame, timestamp); + } + frame_count += 1; + Ok(Some((texture.clone(), frame_time))) + }, + |output_sample| { + let mut output = output.lock().unwrap(); + let _ = muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| format!("WriteSample: {e}")); + Ok(()) + }, + ) + .context("run camera encoder")?; + } + + info!( + "Windows camera encoder finished (hardware): {} frames encoded", + frame_count + ); + Ok(()) + } + either::Right(mut encoder) => { + info!( + "Windows camera encoder started (software): {}x{} -> {}x{} @ {}fps", + video_config.width, + video_config.height, + output_width, + output_height, + frame_rate + ); + + let mut first_timestamp: Option = None; + let mut frame_count = 0u64; + + let mut process_frame = + |frame: NativeCameraFrame, + timestamp: Duration| + -> anyhow::Result> { + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + + let ffmpeg_frame = camera_frame_to_ffmpeg(&frame)?; + + let Ok(mut output_guard) = output.lock() else { + return Ok(None); + }; + + encoder + .queue_frame(ffmpeg_frame, relative, &mut output_guard) + .context("queue camera frame")?; + + Ok(Some(relative)) + }; + + if process_frame(first_frame.0, first_frame.1)?.is_some() { + frame_count += 1; + } + + while let Ok(Some((frame, timestamp))) = video_rx.recv() { + if process_frame(frame, timestamp)?.is_some() { frame_count += 1; - Ok(Some((texture.clone(), frame_time))) - }, - |output_sample| { - let mut output = output.lock().unwrap(); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); - Ok(()) - }, - ) - .context("run camera encoder")?; - } + if frame_count.is_multiple_of(30) { + debug!( + "Windows camera encoder (software): processed {} frames", + frame_count + ); + } + } + } - info!( - "Windows camera encoder finished: {} frames encoded", - frame_count - ); - Ok(()) + info!( + "Windows camera encoder finished (software): {} frames encoded", + frame_count + ); + Ok(()) + } + } }); } @@ -738,6 +839,60 @@ fn convert_uyvy_to_yuyv(src: &[u8], width: u32, height: u32) -> Vec { dst } +pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result { + use cap_mediafoundation_utils::IMFMediaBufferExt; + + let ffmpeg_format = match frame.pixel_format { + cap_camera_windows::PixelFormat::NV12 => ffmpeg::format::Pixel::NV12, + cap_camera_windows::PixelFormat::YUYV422 => ffmpeg::format::Pixel::YUYV422, + cap_camera_windows::PixelFormat::UYVY422 => ffmpeg::format::Pixel::UYVY422, + other => anyhow::bail!("Unsupported camera pixel format: {:?}", other), + }; + + let buffer_guard = frame + .buffer + .lock() + .map_err(|_| anyhow!("Failed to lock camera buffer"))?; + let lock = buffer_guard + .lock() + .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; + let data = &*lock; + + let converted_data: Option>; + let (final_data, final_format): (&[u8], ffmpeg::format::Pixel) = + if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { + converted_data = Some(convert_uyvy_to_yuyv(data, frame.width, frame.height)); + ( + converted_data.as_ref().unwrap(), + ffmpeg::format::Pixel::YUYV422, + ) + } else { + (data, ffmpeg_format) + }; + + let mut ffmpeg_frame = ffmpeg::frame::Video::new(final_format, frame.width, frame.height); + + match final_format { + ffmpeg::format::Pixel::NV12 => { + let y_size = (frame.width * frame.height) as usize; + let uv_size = y_size / 2; + if final_data.len() >= y_size + uv_size { + ffmpeg_frame.data_mut(0)[..y_size].copy_from_slice(&final_data[..y_size]); + ffmpeg_frame.data_mut(1)[..uv_size].copy_from_slice(&final_data[y_size..]); + } + } + ffmpeg::format::Pixel::YUYV422 => { + let size = (frame.width * frame.height * 2) as usize; + if final_data.len() >= size { + ffmpeg_frame.data_mut(0)[..size].copy_from_slice(&final_data[..size]); + } + } + _ => {} + } + + Ok(ffmpeg_frame) +} + pub fn upload_mf_buffer_to_texture( device: &ID3D11Device, frame: &NativeCameraFrame, diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 67878cf3a2..88462f1d67 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -147,6 +147,7 @@ pub struct WindowsSegmentedCameraMuxer { video_config: VideoInfo, output_height: Option, + encoder_preferences: crate::capture_pipeline::EncoderPreferences, pause: PauseTracker, frame_drops: FrameDropTracker, @@ -155,6 +156,7 @@ pub struct WindowsSegmentedCameraMuxer { pub struct WindowsSegmentedCameraMuxerConfig { pub output_height: Option, pub segment_duration: Duration, + pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, } impl Default for WindowsSegmentedCameraMuxerConfig { @@ -162,6 +164,7 @@ impl Default for WindowsSegmentedCameraMuxerConfig { Self { output_height: None, segment_duration: Duration::from_secs(3), + encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), } } } @@ -195,6 +198,7 @@ impl Muxer for WindowsSegmentedCameraMuxer { current_state: None, video_config, output_height: config.output_height, + encoder_preferences: config.encoder_preferences, pause: PauseTracker::new(pause_flag), frame_drops: FrameDropTracker::new(), }) @@ -341,6 +345,8 @@ impl WindowsSegmentedCameraMuxer { } fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> { + use crate::output_pipeline::win::camera_frame_to_ffmpeg; + let segment_path = self.current_segment_path(); let input_size = SizeInt32 { @@ -361,6 +367,8 @@ impl WindowsSegmentedCameraMuxer { let frame_rate = self.video_config.fps(); let bitrate_multiplier = 0.2f32; let input_format = first_frame.dxgi_format(); + let video_config = self.video_config; + let encoder_preferences = self.encoder_preferences.clone(); let (video_tx, video_rx) = sync_channel::>(30); let (ready_tx, ready_rx) = sync_channel::>(1); @@ -381,85 +389,154 @@ impl WindowsSegmentedCameraMuxer { } }; - let encoder_result = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ); - - let (mut encoder, mut muxer) = match encoder_result { - Ok(encoder) => { - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - let _ = ready_tx.send(Err(anyhow!( - "Failed to lock output mutex: {poisoned}" - ))); - return Err(anyhow!( - "Failed to lock output mutex: {}", - poisoned - )); - } - }; + let encoder = (|| { + let fallback = |reason: Option| { + encoder_preferences.force_software_only(); + if let Some(reason) = reason.as_ref() { + error!( + "Falling back to software H264 encoder for camera segment: {reason}" + ); + } else { + info!("Using software H264 encoder for camera segment"); + } - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) + let mut output_guard = match output_clone.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return Err(anyhow!( + "CameraSegmentSoftwareEncoder: failed to lock output mutex: {}", + poisoned + )); + } }; - match muxer { - Ok(muxer) => (encoder, muxer), - Err(err) => { - let _ = - ready_tx.send(Err(anyhow!("Failed to create muxer: {err}"))); - return Err(anyhow!("Failed to create muxer: {err}")); + cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) + .with_output_size(output_width, output_height) + .and_then(|builder| builder.build(&mut output_guard)) + .map(either::Right) + .map_err(|e| anyhow!("CameraSegmentSoftwareEncoder/{e}")) + }; + + if encoder_preferences.should_force_software() { + return fallback(None); + } + + match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + &d3d_device, + input_format, + input_size, + output_size, + frame_rate, + bitrate_multiplier, + ) { + Ok(encoder) => { + let muxer = { + let mut output_guard = match output_clone.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return fallback(Some(format!( + "Failed to lock output mutex: {poisoned}" + ))); + } + }; + + cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output_guard, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: output_width, + height: output_height, + fps: frame_rate, + bitrate: encoder.bitrate(), + fragmented: false, + frag_duration_us: 0, + }, + ) + }; + + match muxer { + Ok(muxer) => Ok(either::Left((encoder, muxer))), + Err(err) => fallback(Some(err.to_string())), } } + Err(err) => fallback(Some(err.to_string())), } - Err(err) => { - let _ = ready_tx.send(Err(anyhow!("Failed to create H264 encoder: {err}"))); - return Err(anyhow!("Failed to create H264 encoder: {err}")); + })(); + + let encoder = match encoder { + Ok(encoder) => { + if ready_tx.send(Ok(())).is_err() { + error!("Failed to send ready signal - receiver dropped"); + return Ok(()); + } + encoder + } + Err(e) => { + error!("Camera segment encoder setup failed: {:#}", e); + let _ = ready_tx.send(Err(anyhow!("{e}"))); + return Err(anyhow!("{e}")); } }; - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } + match encoder { + either::Left((mut encoder, mut muxer)) => { + info!( + "Camera segment encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", + input_format, + input_size.Width, + input_size.Height, + output_size.Width, + output_size.Height, + frame_rate + ); - info!( - "Camera segment encoder started: {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available for segment"); - return Ok(None); - }; + let mut first_timestamp: Option = None; + + encoder + .run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more camera frames available for segment"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + + let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; + Ok(Some((texture, duration_to_timespan(relative)))) + }, + |output_sample| { + let mut output = output_clone.lock().unwrap(); + muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| { + windows::core::Error::new( + windows::core::HRESULT(-1), + format!("WriteSample: {e}"), + ) + }) + }, + ) + .context("run camera encoder for segment") + } + either::Right(mut encoder) => { + info!( + "Camera segment encoder started (software): {}x{} -> {}x{} @ {}fps", + video_config.width, + video_config.height, + output_width, + output_height, + frame_rate + ); + let mut first_timestamp: Option = None; + + while let Ok(Some((frame, timestamp))) = video_rx.recv() { let relative = if let Some(first) = first_timestamp { timestamp.checked_sub(first).unwrap_or(Duration::ZERO) } else { @@ -467,22 +544,28 @@ impl WindowsSegmentedCameraMuxer { Duration::ZERO }; - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }, - |output_sample| { - let mut output = output_clone.lock().unwrap(); - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("WriteSample: {e}"), - ) - }) - }, - ) - .context("run camera encoder for segment") + let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) { + Ok(f) => f, + Err(e) => { + warn!("Failed to convert camera frame: {e}"); + continue; + } + }; + + let Ok(mut output_guard) = output_clone.lock() else { + continue; + }; + + if let Err(e) = + encoder.queue_frame(ffmpeg_frame, relative, &mut output_guard) + { + warn!("Failed to queue camera frame: {e}"); + } + } + + Ok(()) + } + } })?; ready_rx diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 0a5a437ab4..325ee7509e 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -874,7 +874,7 @@ async fn create_segment_pipeline( start_time, fragmented, #[cfg(windows)] - encoder_preferences, + encoder_preferences.clone(), ) .instrument(error_span!("screen-out")) .await @@ -912,14 +912,20 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_video::(camera_feed) .with_timestamps(start_time) - .build::(WindowsSegmentedCameraMuxerConfig::default()) + .build::(WindowsSegmentedCameraMuxerConfig { + encoder_preferences: encoder_preferences.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await } else { OutputPipeline::builder(dir.join("camera.mp4")) .with_video::(camera_feed) .with_timestamps(start_time) - .build::(WindowsCameraMuxerConfig::default()) + .build::(WindowsCameraMuxerConfig { + encoder_preferences: encoder_preferences.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await }; diff --git a/crates/scap-direct3d/Cargo.toml b/crates/scap-direct3d/Cargo.toml index d19e4e8237..ace537f6e0 100644 --- a/crates/scap-direct3d/Cargo.toml +++ b/crates/scap-direct3d/Cargo.toml @@ -41,4 +41,5 @@ workspace = true [dependencies] thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index 77ebf61408..d671d86914 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -22,19 +22,20 @@ use windows::{ Win32::{ Foundation::HMODULE, Graphics::{ - Direct3D::D3D_DRIVER_TYPE_HARDWARE, + Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, - D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, - ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ, + D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, + ID3D11DeviceContext, ID3D11Texture2D, }, Dxgi::{ Common::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC, }, - IDXGIDevice, + DXGI_ERROR_UNSUPPORTED, IDXGIDevice, }, }, System::WinRT::Direct3D11::{ @@ -157,6 +158,43 @@ pub fn is_supported() -> windows::core::Result { )? && GraphicsCaptureSession::IsSupported()?) } +fn create_d3d_device_with_type( + driver_type: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, + device: *mut Option, +) -> windows::core::Result<()> { + unsafe { + D3D11CreateDevice( + None, + driver_type, + HMODULE::default(), + flags, + None, + D3D11_SDK_VERSION, + Some(device), + None, + None, + ) + } +} + +fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device, bool)> { + let mut device = None; + let flags = D3D11_CREATE_DEVICE_FLAG::default(); + + let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + + match result { + Ok(()) => Ok((device.unwrap(), false)), + Err(e) if e.code() == DXGI_ERROR_UNSUPPORTED => { + tracing::info!("Hardware D3D11 device unavailable, attempting WARP fallback"); + create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device)?; + Ok((device.unwrap(), true)) + } + Err(e) => Err(e), + } +} + #[derive(Clone, Default, Debug)] pub struct Settings { pub is_border_required: Option, @@ -192,6 +230,12 @@ impl Settings { #[derive(Clone, Debug, thiserror::Error)] pub enum NewCapturerError { + #[error("Screen capture requires Windows 10 version 1903 (build 18362) or later")] + WindowsVersionTooOld, + #[error( + "Windows Graphics Capture API is disabled or unavailable. This may be due to group policy or missing system components." + )] + GraphicsCaptureDisabled, #[error("NotSupported")] NotSupported, #[error("BorderNotSupported")] @@ -232,6 +276,7 @@ pub struct Capturer { frame_pool: Direct3D11CaptureFramePool, frame_arrived_token: i64, stop_flag: Arc, + is_using_warp: bool, } impl Capturer { @@ -240,10 +285,20 @@ impl Capturer { settings: Settings, mut callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, - mut d3d_device: Option, + d3d_device: Option, ) -> Result { - if !is_supported()? { - return Err(NewCapturerError::NotSupported); + let api_present = ApiInformation::IsApiContractPresentByMajor( + &HSTRING::from("Windows.Foundation.UniversalApiContract"), + 8, + ) + .unwrap_or(false); + + if !api_present { + return Err(NewCapturerError::WindowsVersionTooOld); + } + + if !GraphicsCaptureSession::IsSupported().unwrap_or(false) { + return Err(NewCapturerError::GraphicsCaptureDisabled); } if settings.is_border_required.is_some() && !Settings::can_is_border_required()? { @@ -260,24 +315,13 @@ impl Capturer { return Err(NewCapturerError::UpdateIntervalNotSupported); } - if d3d_device.is_none() { - unsafe { - D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - Default::default(), - None, - D3D11_SDK_VERSION, - Some(&mut d3d_device), - None, - None, - ) - } - .map_err(NewCapturerError::CreateDevice)?; - } + let (d3d_device, is_using_warp) = if let Some(device) = d3d_device { + (device, false) + } else { + create_d3d_device_with_warp_fallback().map_err(NewCapturerError::CreateDevice)? + }; - let (d3d_device, d3d_context) = d3d_device + let (d3d_device, d3d_context) = Some(d3d_device) .map(|d| unsafe { d.GetImmediateContext() }.map(|v| (d, v))) .transpose() .map_err(NewCapturerError::Context)? @@ -433,6 +477,12 @@ impl Capturer { ) .map_err(NewCapturerError::RegisterClosed)?; + if is_using_warp { + tracing::warn!( + "Hardware GPU unavailable, using WARP software rasterizer for screen capture" + ); + } + Ok(Capturer { settings, d3d_device, @@ -441,9 +491,14 @@ impl Capturer { frame_pool, frame_arrived_token, stop_flag, + is_using_warp, }) } + pub fn is_using_software_rendering(&self) -> bool { + self.is_using_warp + } + pub fn settings(&self) -> &Settings { &self.settings } From 5d4bc71b77f32a72b20aad763848f2b4c74d4f5b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 15:49:08 +0000 Subject: [PATCH 07/26] Add Windows version detection and minimum check --- crates/scap-direct3d/Cargo.toml | 1 + crates/scap-direct3d/src/lib.rs | 23 ++- crates/scap-direct3d/src/windows_version.rs | 174 ++++++++++++++++++++ 3 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 crates/scap-direct3d/src/windows_version.rs diff --git a/crates/scap-direct3d/Cargo.toml b/crates/scap-direct3d/Cargo.toml index ace537f6e0..f95306ec84 100644 --- a/crates/scap-direct3d/Cargo.toml +++ b/crates/scap-direct3d/Cargo.toml @@ -29,6 +29,7 @@ windows = { workspace = true, features = [ "Win32_System_Variant", "Storage_Search", "Storage_Streams", + "Win32_System_SystemInformation", ] } [dev-dependencies] diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index d671d86914..1a2dffaab5 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -1,7 +1,9 @@ -// a whole bunch of credit to https://github.com/NiiightmareXD/windows-capture - #![cfg(windows)] +mod windows_version; + +pub use windows_version::WindowsVersion; + use std::{ sync::{ Arc, Mutex, @@ -287,6 +289,23 @@ impl Capturer { mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, d3d_device: Option, ) -> Result { + if let Some(version) = WindowsVersion::detect() { + tracing::debug!( + version = %version.display_name(), + meets_requirements = version.meets_minimum_requirements(), + "Initializing screen capture" + ); + + if !version.meets_minimum_requirements() { + tracing::error!( + version = %version.display_name(), + required = "Windows 10 version 1903 (build 18362)", + "Windows version does not meet minimum requirements" + ); + return Err(NewCapturerError::WindowsVersionTooOld); + } + } + let api_present = ApiInformation::IsApiContractPresentByMajor( &HSTRING::from("Windows.Foundation.UniversalApiContract"), 8, diff --git a/crates/scap-direct3d/src/windows_version.rs b/crates/scap-direct3d/src/windows_version.rs new file mode 100644 index 0000000000..39cdd89c86 --- /dev/null +++ b/crates/scap-direct3d/src/windows_version.rs @@ -0,0 +1,174 @@ +#![cfg(windows)] + +use std::sync::OnceLock; +use windows::Win32::System::SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}; + +static DETECTED_VERSION: OnceLock> = OnceLock::new(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WindowsVersion { + pub major: u32, + pub minor: u32, + pub build: u32, +} + +impl WindowsVersion { + pub fn detect() -> Option { + *DETECTED_VERSION.get_or_init(|| detect_version_internal()) + } + + pub fn meets_minimum_requirements(&self) -> bool { + self.major > 10 || (self.major == 10 && self.build >= 18362) + } + + pub fn supports_border_control(&self) -> bool { + self.build >= 22000 + } + + pub fn is_windows_11(&self) -> bool { + self.build >= 22000 + } + + pub fn display_name(&self) -> String { + if self.build >= 22000 { + format!("Windows 11 (Build {})", self.build) + } else if self.major == 10 { + format!("Windows 10 (Build {})", self.build) + } else { + format!( + "Windows {}.{} (Build {})", + self.major, self.minor, self.build + ) + } + } +} + +fn detect_version_internal() -> Option { + unsafe { + let mut info = OSVERSIONINFOEXW { + dwOSVersionInfoSize: std::mem::size_of::() as u32, + ..Default::default() + }; + + let info_ptr = &mut info as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; + + #[allow(deprecated)] + if GetVersionExW(info_ptr).is_ok() { + let version = WindowsVersion { + major: info.dwMajorVersion, + minor: info.dwMinorVersion, + build: info.dwBuildNumber, + }; + + tracing::debug!( + major = version.major, + minor = version.minor, + build = version.build, + display_name = %version.display_name(), + "Detected Windows version" + ); + + return Some(version); + } + + let mut basic_info = OSVERSIONINFOW { + dwOSVersionInfoSize: std::mem::size_of::() as u32, + ..Default::default() + }; + + #[allow(deprecated)] + if GetVersionExW(&mut basic_info).is_ok() { + let version = WindowsVersion { + major: basic_info.dwMajorVersion, + minor: basic_info.dwMinorVersion, + build: basic_info.dwBuildNumber, + }; + + tracing::debug!( + major = version.major, + minor = version.minor, + build = version.build, + display_name = %version.display_name(), + "Detected Windows version (basic)" + ); + + return Some(version); + } + + tracing::warn!("Failed to detect Windows version"); + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_detect_returns_some() { + let version = WindowsVersion::detect(); + assert!(version.is_some(), "Should detect Windows version"); + } + + #[test] + fn test_version_requirements() { + let old_version = WindowsVersion { + major: 10, + minor: 0, + build: 17000, + }; + assert!(!old_version.meets_minimum_requirements()); + + let min_version = WindowsVersion { + major: 10, + minor: 0, + build: 18362, + }; + assert!(min_version.meets_minimum_requirements()); + + let new_version = WindowsVersion { + major: 10, + minor: 0, + build: 19041, + }; + assert!(new_version.meets_minimum_requirements()); + } + + #[test] + fn test_windows_11_detection() { + let win10 = WindowsVersion { + major: 10, + minor: 0, + build: 19045, + }; + assert!(!win10.is_windows_11()); + assert!(!win10.supports_border_control()); + + let win11 = WindowsVersion { + major: 10, + minor: 0, + build: 22000, + }; + assert!(win11.is_windows_11()); + assert!(win11.supports_border_control()); + } + + #[test] + fn test_display_name() { + let win10 = WindowsVersion { + major: 10, + minor: 0, + build: 19045, + }; + assert!(win10.display_name().contains("Windows 10")); + assert!(win10.display_name().contains("19045")); + + let win11 = WindowsVersion { + major: 10, + minor: 0, + build: 22631, + }; + assert!(win11.display_name().contains("Windows 11")); + assert!(win11.display_name().contains("22631")); + } +} From 6f50aedce8cfc98c351beebc8ddc5a46740018a0 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 15:49:15 +0000 Subject: [PATCH 08/26] Add support for BGRA, RGBA, and P010LE formats in D3D11 --- crates/frame-converter/src/d3d11.rs | 171 ++++++++++++++++++++++++++-- 1 file changed, 159 insertions(+), 12 deletions(-) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 9f0c44e377..35d81f0fdd 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -8,6 +8,7 @@ use std::{ OnceLock, atomic::{AtomicBool, AtomicU64, Ordering}, }, + time::Instant, }; use windows::{ Win32::{ @@ -27,7 +28,10 @@ use windows::{ ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, }, Dxgi::{ - Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_YUY2}, + Common::{ + DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_P010, + DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_YUY2, + }, CreateDXGIFactory1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, IDXGIAdapter, IDXGIDevice, IDXGIFactory1, IDXGIResource1, }, @@ -149,18 +153,16 @@ struct D3D11Resources { pub struct D3D11Converter { resources: Mutex, - #[allow(dead_code)] input_format: Pixel, output_format: Pixel, - #[allow(dead_code)] input_width: u32, - #[allow(dead_code)] input_height: u32, output_width: u32, output_height: u32, gpu_info: GpuInfo, conversion_count: AtomicU64, verified_gpu_usage: AtomicBool, + total_conversion_time_ns: AtomicU64, } fn get_gpu_info(device: &ID3D11Device) -> Result { @@ -373,6 +375,7 @@ impl D3D11Converter { gpu_info, conversion_count: AtomicU64::new(0), verified_gpu_usage: AtomicBool::new(false), + total_conversion_time_ns: AtomicU64::new(0), }) } @@ -396,15 +399,56 @@ impl D3D11Converter { let resources = self.resources.lock(); resources.input_shared_handle.is_some() && resources.output_shared_handle.is_some() } + + pub fn average_conversion_time_ms(&self) -> Option { + let count = self.conversion_count.load(Ordering::Relaxed); + if count == 0 { + return None; + } + let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); + Some((total_ns as f64 / count as f64) / 1_000_000.0) + } + + pub fn format_info(&self) -> (Pixel, Pixel, u32, u32, u32, u32) { + ( + self.input_format, + self.output_format, + self.input_width, + self.input_height, + self.output_width, + self.output_height, + ) + } +} + +pub fn supported_input_formats() -> &'static [Pixel] { + &[ + Pixel::NV12, + Pixel::YUYV422, + Pixel::BGRA, + Pixel::RGBA, + Pixel::P010LE, + ] +} + +pub fn is_format_supported(pixel: Pixel) -> bool { + pixel_to_dxgi(pixel).is_ok() } impl FrameConverter for D3D11Converter { fn convert(&self, input: frame::Video) -> Result { + let start = Instant::now(); let count = self.conversion_count.fetch_add(1, Ordering::Relaxed); if count == 0 { tracing::info!( - "D3D11 converter first frame: converting on GPU {} ({})", + "D3D11 converter first frame: {:?} {}x{} -> {:?} {}x{} on GPU {} ({})", + self.input_format, + self.input_width, + self.input_height, + self.output_format, + self.output_width, + self.output_height, self.gpu_info.description, self.gpu_info.vendor_name() ); @@ -543,6 +587,25 @@ impl FrameConverter for D3D11Converter { resources.context.Unmap(&resources.staging_output, 0); output.set_pts(pts); + + let elapsed_ns = start.elapsed().as_nanos() as u64; + self.total_conversion_time_ns + .fetch_add(elapsed_ns, Ordering::Relaxed); + + let frame_count = count + 1; + if frame_count > 0 && frame_count % 300 == 0 { + let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); + let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; + tracing::debug!( + "D3D11 converter: {:.2}ms avg over {} frames ({:?} -> {:?}) on {}", + avg_ms, + frame_count, + self.input_format, + self.output_format, + self.gpu_info.description + ); + } + Ok(output) } } @@ -568,10 +631,25 @@ fn pixel_to_dxgi(pixel: Pixel) -> Result { match pixel { Pixel::NV12 => Ok(DXGI_FORMAT_NV12), Pixel::YUYV422 => Ok(DXGI_FORMAT_YUY2), + Pixel::BGRA => Ok(DXGI_FORMAT_B8G8R8A8_UNORM), + Pixel::RGBA => Ok(DXGI_FORMAT_R8G8B8A8_UNORM), + Pixel::P010LE => Ok(DXGI_FORMAT_P010), _ => Err(ConvertError::UnsupportedFormat(pixel, Pixel::NV12)), } } +#[allow(dead_code)] +fn dxgi_format_name(format: DXGI_FORMAT) -> &'static str { + match format { + DXGI_FORMAT_NV12 => "NV12", + DXGI_FORMAT_YUY2 => "YUY2", + DXGI_FORMAT_B8G8R8A8_UNORM => "BGRA", + DXGI_FORMAT_R8G8B8A8_UNORM => "RGBA", + DXGI_FORMAT_P010 => "P010", + _ => "Unknown", + } +} + fn create_texture( device: &ID3D11Device, width: u32, @@ -687,6 +765,7 @@ fn extract_shared_handle(texture: &ID3D11Texture2D) -> Option { unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: usize) { let height = frame.height() as usize; + let width = frame.width() as usize; let format = frame.format(); match format { @@ -696,7 +775,7 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u ptr::copy_nonoverlapping( frame.data(0).as_ptr().add(y * frame.stride(0)), dst.add(y * dst_stride), - frame.width() as usize, + width, ); } } @@ -706,13 +785,37 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u ptr::copy_nonoverlapping( frame.data(1).as_ptr().add(y * frame.stride(1)), dst.add(uv_offset + y * dst_stride), - frame.width() as usize, + width, ); } } } Pixel::YUYV422 => { - let row_bytes = frame.width() as usize * 2; + let row_bytes = width * 2; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + frame.data(0).as_ptr().add(y * frame.stride(0)), + dst.add(y * dst_stride), + row_bytes, + ); + } + } + } + Pixel::BGRA | Pixel::RGBA => { + let row_bytes = width * 4; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + frame.data(0).as_ptr().add(y * frame.stride(0)), + dst.add(y * dst_stride), + row_bytes, + ); + } + } + } + Pixel::P010LE => { + let row_bytes = width * 2; for y in 0..height { unsafe { ptr::copy_nonoverlapping( @@ -722,6 +825,16 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u ); } } + let uv_offset = height * dst_stride; + for y in 0..height / 2 { + unsafe { + ptr::copy_nonoverlapping( + frame.data(1).as_ptr().add(y * frame.stride(1)), + dst.add(uv_offset + y * dst_stride), + row_bytes, + ); + } + } } _ => {} } @@ -729,6 +842,7 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut frame::Video) { let height = frame.height() as usize; + let width = frame.width() as usize; let format = frame.format(); match format { @@ -738,7 +852,7 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr ptr::copy_nonoverlapping( src.add(y * src_stride), frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - frame.width() as usize, + width, ); } } @@ -748,14 +862,37 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr ptr::copy_nonoverlapping( src.add(uv_offset + y * src_stride), frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), - frame.width() as usize, + width, ); } } } Pixel::YUYV422 => { - let bytes_per_pixel = 2; - let row_bytes = frame.width() as usize * bytes_per_pixel; + let row_bytes = width * 2; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + src.add(y * src_stride), + frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), + row_bytes, + ); + } + } + } + Pixel::BGRA | Pixel::RGBA => { + let row_bytes = width * 4; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + src.add(y * src_stride), + frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), + row_bytes, + ); + } + } + } + Pixel::P010LE => { + let row_bytes = width * 2; for y in 0..height { unsafe { ptr::copy_nonoverlapping( @@ -765,6 +902,16 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr ); } } + let uv_offset = height * src_stride; + for y in 0..height / 2 { + unsafe { + ptr::copy_nonoverlapping( + src.add(uv_offset + y * src_stride), + frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), + row_bytes, + ); + } + } } _ => {} } From 0d71888d96da285b2af0dcfedbaf53f97db00dfc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:06:56 +0000 Subject: [PATCH 09/26] Add system diagnostics collection and UI display --- .claude/settings.local.json | 3 +- apps/desktop/src-tauri/src/lib.rs | 7 + .../(window-chrome)/settings/feedback.tsx | 100 +++++++++- apps/desktop/src/utils/tauri.ts | 6 + crates/recording/src/diagnostics.rs | 177 ++++++++++++++++++ crates/recording/src/lib.rs | 1 + 6 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 crates/recording/src/diagnostics.rs diff --git a/.claude/settings.local.json b/.claude/settings.local.json index a9944f0f3d..bb84c28cf6 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -42,7 +42,8 @@ "Bash(cargo tree:*)", "WebFetch(domain:github.com)", "WebFetch(domain:docs.rs)", - "WebFetch(domain:gix.github.io)" + "WebFetch(domain:gix.github.io)", + "Bash(cargo clean:*)" ], "deny": [], "ask": [] diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index f5c19546d3..2b1d08cff5 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -420,6 +420,12 @@ async fn upload_logs(app_handle: AppHandle) -> Result<(), String> { logging::upload_log_file(&app_handle).await } +#[tauri::command] +#[specta::specta] +fn get_system_diagnostics() -> cap_recording::diagnostics::SystemDiagnostics { + cap_recording::diagnostics::collect_diagnostics() +} + #[tauri::command] #[specta::specta] #[instrument(skip(app_handle, state))] @@ -2337,6 +2343,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { set_camera_input, recording_settings::set_recording_mode, upload_logs, + get_system_diagnostics, recording::start_recording, recording::stop_recording, recording::pause_recording, diff --git a/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx b/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx index f81059faac..ae984036cf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx @@ -2,10 +2,10 @@ import { Button } from "@cap/ui-solid"; import { action, useAction, useSubmission } from "@solidjs/router"; import { getVersion } from "@tauri-apps/api/app"; import { type as ostype } from "@tauri-apps/plugin-os"; -import { createSignal } from "solid-js"; +import { createResource, createSignal, For, Show } from "solid-js"; import toast from "solid-toast"; -import { commands } from "~/utils/tauri"; +import { commands, type SystemDiagnostics } from "~/utils/tauri"; import { apiClient, protectedHeaders } from "~/utils/web-api"; const sendFeedbackAction = action(async (feedback: string) => { @@ -18,9 +18,19 @@ const sendFeedbackAction = action(async (feedback: string) => { return response.body; }); +async function fetchDiagnostics(): Promise { + try { + return await commands.getSystemDiagnostics(); + } catch (e) { + console.error("Failed to fetch diagnostics:", e); + return null; + } +} + export default function FeedbackTab() { const [feedback, setFeedback] = createSignal(""); const [uploadingLogs, setUploadingLogs] = createSignal(false); + const [diagnostics] = createResource(fetchDiagnostics); const submission = useSubmission(sendFeedbackAction); const sendFeedback = useAction(sendFeedbackAction); @@ -107,6 +117,92 @@ export default function FeedbackTab() { {uploadingLogs() ? "Uploading..." : "Upload Logs"} + +
+

+ System Information +

+ + Loading system information... +

+ } + > + {(diag) => ( +
+ + {(ver) => ( +
+

Operating System

+

+ {ver().displayName} +

+
+ )} + + + + {(gpu) => ( +
+

Graphics

+

+ {gpu().description} ({gpu().vendor},{" "} + {gpu().dedicatedVideoMemoryMb} MB VRAM) +

+
+ )} + + +
+

Capture Support

+
+ + Graphics Capture:{" "} + {diag().graphicsCaptureSupported + ? "Supported" + : "Not Supported"} + + + D3D11 Video:{" "} + {diag().d3D11VideoProcessorAvailable + ? "Available" + : "Unavailable"} + +
+
+ + 0}> +
+

Available Encoders

+
+ + {(encoder) => ( + + {encoder} + + )} + +
+
+ +
+ )} + +
diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 88b4042ec5..1afc1a0109 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -17,6 +17,9 @@ async setRecordingMode(mode: RecordingMode) : Promise { async uploadLogs() : Promise { return await TAURI_INVOKE("upload_logs"); }, +async getSystemDiagnostics() : Promise { + return await TAURI_INVOKE("get_system_diagnostics"); +}, async startRecording(inputs: StartRecordingInputs) : Promise { return await TAURI_INVOKE("start_recording", { inputs }); }, @@ -420,6 +423,7 @@ quality: number | null; * Whether to prioritize speed over quality (default: false) */ fast: boolean | null } +export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number } export type HapticPattern = "alignment" | "levelChange" | "generic" export type HapticPerformanceTime = "default" | "now" | "drawCompleted" export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; shift: boolean } @@ -494,6 +498,7 @@ export type StartRecordingInputs = { capture_target: ScreenCaptureTarget; captur export type StereoMode = "stereo" | "monoL" | "monoR" export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments } export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsRemux" } | { status: "Failed"; error: string } | { status: "Complete" } +export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean } export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } export type TextSegment = { start: number; end: number; enabled?: boolean; content?: string; center?: XY; size?: XY; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number } export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[] } @@ -510,6 +515,7 @@ export type VideoUploadInfo = { id: string; link: string; config: S3UploadMeta } export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: string | null; windowTitle?: string | null } export type WindowId = string export type WindowUnderCursor = { id: WindowId; app_name: string; bounds: LogicalBounds } +export type WindowsVersionInfo = { major: number; minor: number; build: number; displayName: string; meetsRequirements: boolean; isWindows11: boolean } export type XY = { x: T; y: T } export type ZoomMode = "auto" | { manual: { x: number; y: number } } export type ZoomSegment = { start: number; end: number; amount: number; mode: ZoomMode } diff --git a/crates/recording/src/diagnostics.rs b/crates/recording/src/diagnostics.rs new file mode 100644 index 0000000000..b304d95d71 --- /dev/null +++ b/crates/recording/src/diagnostics.rs @@ -0,0 +1,177 @@ +#[cfg(target_os = "windows")] +mod windows_impl { + use serde::Serialize; + use specta::Type; + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct WindowsVersionInfo { + pub major: u32, + pub minor: u32, + pub build: u32, + pub display_name: String, + pub meets_requirements: bool, + pub is_windows_11: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct GpuInfoDiag { + pub vendor: String, + pub description: String, + pub dedicated_video_memory_mb: f64, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct SystemDiagnostics { + pub windows_version: Option, + pub gpu_info: Option, + pub available_encoders: Vec, + pub graphics_capture_supported: bool, + pub d3d11_video_processor_available: bool, + } + + pub fn collect_diagnostics() -> SystemDiagnostics { + let windows_version = get_windows_version_info(); + let gpu_info = get_gpu_info(); + let available_encoders = get_available_encoders(); + let graphics_capture_supported = check_graphics_capture_support(); + let d3d11_video_processor_available = check_d3d11_video_processor(); + + tracing::info!("System Diagnostics:"); + if let Some(ref ver) = windows_version { + tracing::info!(" Windows: {}", ver.display_name); + } + if let Some(ref gpu) = gpu_info { + tracing::info!(" GPU: {} ({})", gpu.description, gpu.vendor); + } + tracing::info!(" Encoders: {:?}", available_encoders); + tracing::info!(" Graphics Capture: {}", graphics_capture_supported); + tracing::info!( + " D3D11 Video Processor: {}", + d3d11_video_processor_available + ); + + SystemDiagnostics { + windows_version, + gpu_info, + available_encoders, + graphics_capture_supported, + d3d11_video_processor_available, + } + } + + fn get_windows_version_info() -> Option { + scap_direct3d::WindowsVersion::detect().map(|v| WindowsVersionInfo { + major: v.major, + minor: v.minor, + build: v.build, + display_name: v.display_name(), + meets_requirements: v.meets_minimum_requirements(), + is_windows_11: v.is_windows_11(), + }) + } + + fn get_gpu_info() -> Option { + cap_frame_converter::detect_primary_gpu().map(|info| GpuInfoDiag { + vendor: info.vendor_name().to_string(), + description: info.description.clone(), + dedicated_video_memory_mb: (info.dedicated_video_memory / (1024 * 1024)) as f64, + }) + } + + fn get_available_encoders() -> Vec { + let candidates = [ + "h264_nvenc", + "h264_qsv", + "h264_amf", + "h264_mf", + "libx264", + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + "libx265", + ]; + + candidates + .iter() + .filter(|name| ffmpeg::encoder::find_by_name(name).is_some()) + .map(|s| s.to_string()) + .collect() + } + + fn check_graphics_capture_support() -> bool { + scap_direct3d::is_supported().unwrap_or(false) + } + + fn check_d3d11_video_processor() -> bool { + use cap_frame_converter::ConversionConfig; + + let test_config = ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + cap_frame_converter::D3D11Converter::new(test_config).is_ok() + } +} + +#[cfg(target_os = "macos")] +mod macos_impl { + use serde::Serialize; + use specta::Type; + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct MacOSVersionInfo { + pub display_name: String, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct SystemDiagnostics { + pub macos_version: Option, + pub available_encoders: Vec, + pub screen_capture_supported: bool, + } + + pub fn collect_diagnostics() -> SystemDiagnostics { + let available_encoders = get_available_encoders(); + + tracing::info!("System Diagnostics:"); + tracing::info!(" Encoders: {:?}", available_encoders); + + SystemDiagnostics { + macos_version: None, + available_encoders, + screen_capture_supported: true, + } + } + + fn get_available_encoders() -> Vec { + let candidates = [ + "h264_videotoolbox", + "libx264", + "hevc_videotoolbox", + "libx265", + ]; + + candidates + .iter() + .filter(|name| ffmpeg::encoder::find_by_name(name).is_some()) + .map(|s| s.to_string()) + .collect() + } +} + +#[cfg(target_os = "windows")] +pub use windows_impl::*; + +#[cfg(target_os = "macos")] +pub use macos_impl::*; diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index e68b92d15e..dc1670b27d 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -1,6 +1,7 @@ pub mod benchmark; mod capture_pipeline; pub mod cursor; +pub mod diagnostics; pub mod feeds; pub mod fragmentation; pub mod instant_recording; From 3fd575eb71374fe5512e84ef12fa89bfc6c7e957 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:40:49 +0000 Subject: [PATCH 10/26] Add comprehensive Windows hardware compatibility tests --- .claude/settings.local.json | 3 +- crates/frame-converter/src/d3d11.rs | 17 +- crates/recording/tests/hardware_compat.rs | 931 ++++++++++++++++++++++ 3 files changed, 942 insertions(+), 9 deletions(-) create mode 100644 crates/recording/tests/hardware_compat.rs diff --git a/.claude/settings.local.json b/.claude/settings.local.json index bb84c28cf6..8e65ddf103 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -43,7 +43,8 @@ "WebFetch(domain:github.com)", "WebFetch(domain:docs.rs)", "WebFetch(domain:gix.github.io)", - "Bash(cargo clean:*)" + "Bash(cargo clean:*)", + "Bash(cargo test:*)" ], "deny": [], "ask": [] diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 35d81f0fdd..34eb84e686 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -18,13 +18,14 @@ use windows::{ Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAP_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_SDK_VERSION, - D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, - D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, - D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_STREAM, - D3D11_VPIV_DIMENSION_TEXTURE2D, D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, - ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, ID3D11VideoContext, - ID3D11VideoDevice, ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, + D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED, + D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, + D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VPIV_DIMENSION_TEXTURE2D, + D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, ID3D11Device, + ID3D11DeviceContext, ID3D11Texture2D, ID3D11VideoContext, ID3D11VideoDevice, + ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, }, Dxgi::{ @@ -708,7 +709,7 @@ fn create_shared_texture( Usage: D3D11_USAGE_DEFAULT, BindFlags: bind_flags, CPUAccessFlags: 0, - MiscFlags: D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0 as u32, + MiscFlags: (D3D11_RESOURCE_MISC_SHARED.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0) as u32, }; let texture = unsafe { diff --git a/crates/recording/tests/hardware_compat.rs b/crates/recording/tests/hardware_compat.rs new file mode 100644 index 0000000000..d5c940caa9 --- /dev/null +++ b/crates/recording/tests/hardware_compat.rs @@ -0,0 +1,931 @@ +#![cfg(target_os = "windows")] + +use std::{collections::HashMap, time::Duration}; + +mod test_utils { + use std::sync::Once; + + static INIT: Once = Once::new(); + + pub fn init_tracing() { + INIT.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive(tracing::Level::DEBUG.into()), + ) + .with_test_writer() + .try_init() + .ok(); + }); + } +} + +#[test] +fn test_software_encoding_always_available() { + test_utils::init_tracing(); + + let libx264 = ffmpeg::encoder::find_by_name("libx264"); + assert!( + libx264.is_some(), + "libx264 software encoder must always be available as ultimate fallback" + ); + + let encoder = libx264.unwrap(); + println!("libx264 encoder available: {}", encoder.description()); +} + +#[test] +fn test_swscale_conversion_works() { + test_utils::init_tracing(); + + let config = cap_frame_converter::ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + let result = cap_frame_converter::create_converter_with_details(config); + assert!( + result.is_ok(), + "Frame converter should always succeed (with swscale fallback)" + ); + + let selection = result.unwrap(); + println!( + "Converter backend: {:?}, fallback reason: {:?}", + selection.backend, selection.fallback_reason + ); +} + +#[test] +fn test_system_diagnostics_collection() { + test_utils::init_tracing(); + + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + + println!("=== System Diagnostics ==="); + + if let Some(ref version) = diagnostics.windows_version { + println!( + "Windows: {} (Build {})", + version.display_name, version.build + ); + println!(" Meets requirements: {}", version.meets_requirements); + println!(" Is Windows 11: {}", version.is_windows_11); + } else { + println!("Windows version: Could not detect"); + } + + if let Some(ref gpu) = diagnostics.gpu_info { + println!( + "GPU: {} ({}) - {} MB VRAM", + gpu.description, gpu.vendor, gpu.dedicated_video_memory_mb + ); + } else { + println!("GPU: No dedicated GPU detected (CPU-only or WARP)"); + } + + println!("Available encoders: {:?}", diagnostics.available_encoders); + println!( + "Graphics Capture supported: {}", + diagnostics.graphics_capture_supported + ); + println!( + "D3D11 Video Processor available: {}", + diagnostics.d3d11_video_processor_available + ); + + assert!( + diagnostics + .available_encoders + .contains(&"libx264".to_string()), + "libx264 must be available" + ); +} + +#[test] +fn test_windows_version_detection() { + test_utils::init_tracing(); + + let version = scap_direct3d::WindowsVersion::detect(); + assert!( + version.is_some(), + "Windows version detection should succeed" + ); + + let version = version.unwrap(); + println!( + "Windows Version: {} (Major: {}, Minor: {}, Build: {})", + version.display_name(), + version.major, + version.minor, + version.build + ); + println!( + "Meets minimum requirements (Windows 10 1903+): {}", + version.meets_minimum_requirements() + ); + println!("Is Windows 11: {}", version.is_windows_11()); + println!( + "Supports border control: {}", + version.supports_border_control() + ); + + let graphics_capture_supported = scap_direct3d::is_supported().unwrap_or(false); + if !version.meets_minimum_requirements() && graphics_capture_supported { + println!( + "Note: GetVersionExW returned version {} but Graphics Capture is supported.", + version.display_name() + ); + println!( + "This is expected - Windows compatibility shims can cause incorrect version reporting." + ); + println!("Feature detection (is_supported) is the reliable method, and it returns true."); + } else if !version.meets_minimum_requirements() { + println!("Warning: Windows version appears to be below requirements."); + println!("If Cap works correctly, this may be a version detection issue."); + } +} + +#[test] +fn test_gpu_detection() { + test_utils::init_tracing(); + + let gpu_info = cap_frame_converter::detect_primary_gpu(); + + if let Some(info) = gpu_info { + println!("=== GPU Information ==="); + println!("Description: {}", info.description); + println!("Vendor: {} (0x{:04X})", info.vendor_name(), info.vendor_id); + println!("Device ID: 0x{:04X}", info.device_id); + println!( + "Dedicated VRAM: {} MB", + info.dedicated_video_memory / (1024 * 1024) + ); + + match info.vendor { + cap_frame_converter::GpuVendor::Nvidia => { + println!(" -> NVIDIA GPU: NVENC encoding expected"); + } + cap_frame_converter::GpuVendor::Amd => { + println!(" -> AMD GPU: AMF encoding expected"); + } + cap_frame_converter::GpuVendor::Intel => { + println!(" -> Intel GPU: QSV encoding expected"); + } + cap_frame_converter::GpuVendor::Qualcomm => { + println!(" -> Qualcomm GPU: Software encoding expected"); + } + cap_frame_converter::GpuVendor::Arm => { + println!(" -> ARM GPU: Software encoding expected"); + } + cap_frame_converter::GpuVendor::Microsoft => { + println!(" -> Microsoft WARP: Software rendering/encoding"); + } + cap_frame_converter::GpuVendor::Unknown(id) => { + println!(" -> Unknown GPU vendor (0x{:04X}): Software fallback", id); + } + } + } else { + println!("No GPU detected - system will use software rendering and encoding"); + } +} + +#[test] +fn test_graphics_capture_support() { + test_utils::init_tracing(); + + let supported = scap_direct3d::is_supported().unwrap_or(false); + println!("Windows Graphics Capture API supported: {}", supported); + + if !supported { + let version = scap_direct3d::WindowsVersion::detect(); + if let Some(v) = version { + if !v.meets_minimum_requirements() { + println!( + " -> Reason: Windows version {} does not meet requirements (need 10.0.18362+)", + v.display_name() + ); + } else { + println!(" -> Reason: Graphics Capture may be disabled by group policy"); + } + } + } +} + +#[test] +fn test_camera_enumeration() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + println!("=== Camera Enumeration ==="); + println!("Found {} camera(s)", cameras.len()); + + for (i, camera) in cameras.iter().enumerate() { + println!("\n--- Camera {} ---", i + 1); + println!("Display name: {}", camera.display_name()); + println!("Device ID: {}", camera.device_id()); + + if let Some(model_id) = camera.model_id() { + println!("Model ID: {}", model_id); + } + + if let Some(formats) = camera.formats() { + println!("Supported formats: {} format(s)", formats.len()); + + let mut format_summary: HashMap> = HashMap::new(); + for format in &formats { + let key = format!("{}x{}", format.width(), format.height()); + format_summary.entry(key).or_default().push(( + format.width(), + format.height(), + format.frame_rate(), + )); + } + + for (resolution, frame_rates) in format_summary.iter() { + let rates: Vec = frame_rates + .iter() + .map(|(_, _, fps)| format!("{:.1}fps", fps)) + .collect(); + println!(" {}: {}", resolution, rates.join(", ")); + } + } else { + println!(" Could not enumerate formats"); + } + } + + if cameras.is_empty() { + println!("\nNo cameras found. This is acceptable for headless/VM environments."); + } +} + +#[test] +fn test_encoder_availability_matrix() { + test_utils::init_tracing(); + + let h264_encoders = [ + ("h264_nvenc", "NVIDIA NVENC"), + ("h264_qsv", "Intel Quick Sync"), + ("h264_amf", "AMD AMF"), + ("h264_mf", "Media Foundation"), + ("libx264", "x264 Software"), + ]; + + let hevc_encoders = [ + ("hevc_nvenc", "NVIDIA NVENC HEVC"), + ("hevc_qsv", "Intel Quick Sync HEVC"), + ("hevc_amf", "AMD AMF HEVC"), + ("hevc_mf", "Media Foundation HEVC"), + ("libx265", "x265 Software"), + ]; + + println!("=== H.264 Encoder Availability ==="); + for (name, description) in h264_encoders { + let available = ffmpeg::encoder::find_by_name(name).is_some(); + let status = if available { "✓" } else { "✗" }; + println!(" {} {} ({})", status, description, name); + } + + println!("\n=== HEVC/H.265 Encoder Availability ==="); + for (name, description) in hevc_encoders { + let available = ffmpeg::encoder::find_by_name(name).is_some(); + let status = if available { "✓" } else { "✗" }; + println!(" {} {} ({})", status, description, name); + } + + let gpu = cap_frame_converter::detect_primary_gpu(); + println!("\n=== Recommended Encoder Priority ==="); + match gpu.map(|g| g.vendor) { + Some(cap_frame_converter::GpuVendor::Nvidia) => { + println!(" NVIDIA detected: h264_nvenc -> h264_mf -> h264_qsv -> h264_amf -> libx264"); + } + Some(cap_frame_converter::GpuVendor::Amd) => { + println!(" AMD detected: h264_amf -> h264_mf -> h264_nvenc -> h264_qsv -> libx264"); + } + Some(cap_frame_converter::GpuVendor::Intel) => { + println!(" Intel detected: h264_qsv -> h264_mf -> h264_nvenc -> h264_amf -> libx264"); + } + _ => { + println!(" Default: h264_nvenc -> h264_qsv -> h264_amf -> h264_mf -> libx264"); + } + } +} + +#[test] +fn test_d3d11_converter_capability() { + test_utils::init_tracing(); + + let test_configs = [ + ( + "BGRA -> NV12 (1080p)", + ffmpeg::format::Pixel::BGRA, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ( + "RGBA -> NV12 (1080p)", + ffmpeg::format::Pixel::RGBA, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ( + "BGRA -> NV12 (4K)", + ffmpeg::format::Pixel::BGRA, + ffmpeg::format::Pixel::NV12, + 3840, + 2160, + ), + ( + "YUYV422 -> NV12 (720p)", + ffmpeg::format::Pixel::YUYV422, + ffmpeg::format::Pixel::NV12, + 1280, + 720, + ), + ( + "NV12 -> NV12 (passthrough)", + ffmpeg::format::Pixel::NV12, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ]; + + println!("=== D3D11 Converter Capability Tests ==="); + + for (name, input, output, width, height) in test_configs { + let config = + cap_frame_converter::ConversionConfig::new(input, width, height, output, width, height); + + match cap_frame_converter::create_converter_with_details(config) { + Ok(result) => { + let hw = if result.converter.is_hardware_accelerated() { + "GPU" + } else { + "CPU" + }; + println!(" ✓ {}: {} ({:?})", name, hw, result.backend); + if let Some(reason) = result.fallback_reason { + println!(" Fallback: {}", reason); + } + } + Err(e) => { + println!(" ✗ {}: Failed - {}", name, e); + } + } + } +} + +#[test] +fn test_supported_pixel_formats() { + test_utils::init_tracing(); + + let formats = [ + (ffmpeg::format::Pixel::NV12, "NV12"), + (ffmpeg::format::Pixel::YUYV422, "YUYV422"), + (ffmpeg::format::Pixel::BGRA, "BGRA"), + (ffmpeg::format::Pixel::RGBA, "RGBA"), + (ffmpeg::format::Pixel::P010LE, "P010LE (10-bit HDR)"), + (ffmpeg::format::Pixel::YUV420P, "YUV420P"), + (ffmpeg::format::Pixel::RGB24, "RGB24"), + ]; + + println!("=== D3D11 Pixel Format Support ==="); + for (format, name) in formats { + let supported = cap_frame_converter::is_format_supported(format); + let status = if supported { "✓" } else { "✗" }; + println!(" {} {}", status, name); + } +} + +#[test] +#[ignore = "Requires NVIDIA GPU - run with --ignored on NVIDIA systems"] +fn test_nvidia_nvenc_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Nvidia) + ) { + println!("Skipping: No NVIDIA GPU detected"); + return; + } + + let nvenc = ffmpeg::encoder::find_by_name("h264_nvenc"); + assert!( + nvenc.is_some(), + "h264_nvenc should be available on NVIDIA systems" + ); + + let nvenc_hevc = ffmpeg::encoder::find_by_name("hevc_nvenc"); + println!( + "NVIDIA NVENC: H.264={}, HEVC={}", + nvenc.is_some(), + nvenc_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires AMD GPU - run with --ignored on AMD systems"] +fn test_amd_amf_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Amd) + ) { + println!("Skipping: No AMD GPU detected"); + return; + } + + let amf = ffmpeg::encoder::find_by_name("h264_amf"); + assert!(amf.is_some(), "h264_amf should be available on AMD systems"); + + let amf_hevc = ffmpeg::encoder::find_by_name("hevc_amf"); + println!( + "AMD AMF: H.264={}, HEVC={}", + amf.is_some(), + amf_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires Intel GPU - run with --ignored on Intel systems"] +fn test_intel_qsv_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Intel) + ) { + println!("Skipping: No Intel GPU detected"); + return; + } + + let qsv = ffmpeg::encoder::find_by_name("h264_qsv"); + assert!( + qsv.is_some(), + "h264_qsv should be available on Intel systems" + ); + + let qsv_hevc = ffmpeg::encoder::find_by_name("hevc_qsv"); + println!( + "Intel Quick Sync: H.264={}, HEVC={}", + qsv.is_some(), + qsv_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires actual camera - run with --ignored when camera is connected"] +fn test_camera_capture_basic() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + if cameras.is_empty() { + println!("No cameras available for capture test"); + return; + } + + let camera = &cameras[0]; + println!("Testing camera: {}", camera.display_name()); + + let formats = camera.formats(); + if formats.is_none() { + println!("Could not get camera formats"); + return; + } + + let formats = formats.unwrap(); + if formats.is_empty() { + println!("Camera has no supported formats"); + return; + } + + let format = formats + .iter() + .find(|f| f.width() == 1280 && f.height() == 720) + .or_else(|| formats.first()) + .cloned() + .unwrap(); + + println!( + "Using format: {}x{} @ {}fps", + format.width(), + format.height(), + format.frame_rate() + ); + + let frame_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let frame_count_clone = frame_count.clone(); + + let handle = camera.start_capturing(format, move |_frame| { + frame_count_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + }); + + match handle { + Ok(handle) => { + std::thread::sleep(Duration::from_secs(2)); + + let frames = frame_count.load(std::sync::atomic::Ordering::Relaxed); + println!("Captured {} frames in 2 seconds", frames); + + let _ = handle.stop_capturing(); + + assert!(frames > 0, "Should have captured at least one frame"); + } + Err(e) => { + println!("Failed to start capture: {:?}", e); + } + } +} + +#[test] +#[ignore = "Requires virtual camera (OBS Virtual Camera) - run with --ignored when available"] +fn test_virtual_camera_detection() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + let virtual_camera_keywords = ["obs", "virtual", "snap", "manycam", "xsplit", "droidcam"]; + + println!("=== Virtual Camera Detection ==="); + for camera in &cameras { + let name_lower = camera.display_name().to_lowercase(); + let is_virtual = virtual_camera_keywords + .iter() + .any(|keyword| name_lower.contains(keyword)); + + if is_virtual { + println!(" [VIRTUAL] {}", camera.display_name()); + } else { + println!(" [PHYSICAL] {}", camera.display_name()); + } + } + + let virtual_count = cameras + .iter() + .filter(|c| { + let name = c.display_name().to_lowercase(); + virtual_camera_keywords + .iter() + .any(|keyword| name.contains(keyword)) + }) + .count(); + + println!( + "\nFound {} virtual camera(s), {} physical camera(s)", + virtual_count, + cameras.len() - virtual_count + ); +} + +#[test] +#[ignore = "Requires capture card (Elgato, etc.) - run with --ignored when available"] +fn test_capture_card_detection() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + let capture_card_keywords = [ + "elgato", + "avermedia", + "magewell", + "blackmagic", + "decklink", + "cam link", + "hd60", + "4k60", + ]; + + println!("=== Capture Card Detection ==="); + for camera in &cameras { + let name_lower = camera.display_name().to_lowercase(); + let is_capture_card = capture_card_keywords + .iter() + .any(|keyword| name_lower.contains(keyword)); + + if is_capture_card { + println!(" [CAPTURE CARD] {}", camera.display_name()); + + if let Some(formats) = camera.formats() { + let max_res = formats.iter().max_by_key(|f| f.width() * f.height()); + if let Some(max) = max_res { + println!( + " Max resolution: {}x{} @ {}fps", + max.width(), + max.height(), + max.frame_rate() + ); + } + } + } + } +} + +#[test] +fn test_hardware_compatibility_summary() { + test_utils::init_tracing(); + + println!("\n╔════════════════════════════════════════════════════════════════╗"); + println!("║ HARDWARE COMPATIBILITY SUMMARY ║"); + println!("╠════════════════════════════════════════════════════════════════╣"); + + let version = scap_direct3d::WindowsVersion::detect(); + let gpu = cap_frame_converter::detect_primary_gpu(); + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + + let windows_status = if diagnostics.graphics_capture_supported { + if let Some(v) = &version { + format!("✓ {} (Graphics Capture OK)", v.display_name()) + } else { + "✓ Graphics Capture supported".to_string() + } + } else if let Some(v) = &version { + format!("✗ {} - Graphics Capture unavailable", v.display_name()) + } else { + "? Unknown".to_string() + }; + println!("║ Windows: {:<52} ║", windows_status); + + let gpu_status = if let Some(g) = gpu { + format!( + "✓ {} ({} MB)", + truncate_string(&g.description, 35), + g.dedicated_video_memory / (1024 * 1024) + ) + } else { + "⚠ No GPU (WARP software rendering)".to_string() + }; + println!("║ GPU: {:<56} ║", gpu_status); + + let capture_status = if diagnostics.graphics_capture_supported { + "✓ Available" + } else { + "✗ Unavailable" + }; + println!("║ Screen Capture: {:<45} ║", capture_status); + + let d3d11_status = if diagnostics.d3d11_video_processor_available { + "✓ GPU accelerated" + } else { + "⚠ CPU fallback (swscale)" + }; + println!("║ Frame Conversion: {:<43} ║", d3d11_status); + + let hw_encoders: Vec<&str> = diagnostics + .available_encoders + .iter() + .filter(|e| !e.starts_with("lib")) + .map(|s| s.as_str()) + .collect(); + let encoder_status = if !hw_encoders.is_empty() { + format!("✓ {} hardware encoder(s)", hw_encoders.len()) + } else { + "⚠ Software only (libx264)".to_string() + }; + println!("║ Encoding: {:<51} ║", encoder_status); + + let cameras: Vec = cap_camera::list_cameras().collect(); + let camera_status = format!("{} camera(s) detected", cameras.len()); + println!("║ Cameras: {:<52} ║", camera_status); + + println!("╠════════════════════════════════════════════════════════════════╣"); + + let all_good = diagnostics.graphics_capture_supported + && diagnostics + .available_encoders + .contains(&"libx264".to_string()); + + if all_good { + println!("║ Status: ✓ SYSTEM COMPATIBLE ║"); + } else { + println!("║ Status: ⚠ COMPATIBILITY ISSUES DETECTED ║"); + } + println!("╚════════════════════════════════════════════════════════════════╝\n"); +} + +fn truncate_string(s: &str, max_len: usize) -> String { + if s.len() <= max_len { + s.to_string() + } else { + format!("{}...", &s[..max_len - 3]) + } +} + +#[test] +#[ignore = "Intensive test - run with --ignored for full validation"] +fn test_frame_conversion_performance() { + test_utils::init_tracing(); + + let config = cap_frame_converter::ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + let result = cap_frame_converter::create_converter_with_details(config.clone()); + if result.is_err() { + println!("Could not create converter: {:?}", result.err()); + return; + } + + let selection = result.unwrap(); + println!( + "Testing converter: {:?} (hardware: {})", + selection.backend, + selection.converter.is_hardware_accelerated() + ); + + let test_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, 1920, 1080); + + let warmup_iterations = 10; + let test_iterations = 100; + + for _ in 0..warmup_iterations { + let frame = test_frame.clone(); + let _ = selection.converter.convert(frame); + } + + let start = std::time::Instant::now(); + for _ in 0..test_iterations { + let frame = test_frame.clone(); + let _ = selection.converter.convert(frame); + } + let elapsed = start.elapsed(); + + let avg_ms = elapsed.as_secs_f64() * 1000.0 / test_iterations as f64; + let fps_capacity = 1000.0 / avg_ms; + + println!( + "Performance: {:.2}ms/frame avg ({:.1} fps capacity)", + avg_ms, fps_capacity + ); + + let target_fps = 60.0; + let target_ms = 1000.0 / target_fps; + if avg_ms < target_ms { + println!("✓ Can sustain {}fps recording", target_fps as u32); + } else { + println!( + "⚠ May struggle with {}fps (need {:.2}ms, got {:.2}ms)", + target_fps as u32, target_ms, avg_ms + ); + } +} + +#[test] +fn test_multi_gpu_detection() { + test_utils::init_tracing(); + + println!("=== Multi-GPU Detection ==="); + println!("Primary GPU detection uses DXGI EnumAdapters(0)"); + + if let Some(gpu) = cap_frame_converter::detect_primary_gpu() { + println!("Primary adapter: {}", gpu.description); + println!("Vendor: {} (0x{:04X})", gpu.vendor_name(), gpu.vendor_id); + + if gpu.dedicated_video_memory < 512 * 1024 * 1024 { + println!("⚠ Low VRAM (<512MB) - software encoding recommended"); + } else if gpu.dedicated_video_memory < 2 * 1024 * 1024 * 1024 { + println!("✓ Adequate VRAM for 1080p recording"); + } else { + println!("✓ Ample VRAM for 4K recording"); + } + } else { + println!("No dedicated GPU found - using integrated or software rendering"); + } +} + +#[test] +fn test_minimum_requirements_check() { + test_utils::init_tracing(); + + let mut requirements_met = true; + let mut warnings = Vec::new(); + + println!("=== Minimum Requirements Check ===\n"); + + println!("Required:"); + + let graphics_capture_supported = scap_direct3d::is_supported().unwrap_or(false); + if graphics_capture_supported { + println!(" ✓ Windows Graphics Capture API"); + } else { + println!(" ✗ Windows Graphics Capture API unavailable"); + requirements_met = false; + } + + let version = scap_direct3d::WindowsVersion::detect(); + if let Some(v) = &version { + if v.meets_minimum_requirements() { + println!( + " ✓ Windows 10 version 1903 or later (reported: {})", + v.display_name() + ); + } else if graphics_capture_supported { + println!( + " ⚠ Windows version reported as {} (may be inaccurate due to compat shims)", + v.display_name() + ); + println!(" Graphics Capture works, so actual version is sufficient."); + } else { + println!( + " ✗ Windows version {} is below minimum (need 10.0.18362+)", + v.display_name() + ); + requirements_met = false; + } + } else { + println!(" ? Could not detect Windows version"); + if !graphics_capture_supported { + requirements_met = false; + } + } + + if ffmpeg::encoder::find_by_name("libx264").is_some() { + println!(" ✓ FFmpeg with libx264 encoder"); + } else { + println!(" ✗ libx264 encoder not available"); + requirements_met = false; + } + + println!("\nRecommended:"); + if cap_frame_converter::detect_primary_gpu().is_some() { + println!(" ✓ Dedicated or integrated GPU"); + } else { + println!(" ⚠ No GPU detected (will use software rendering)"); + warnings.push("Performance may be reduced without GPU acceleration"); + } + + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + let hw_encoders: Vec<&str> = diagnostics + .available_encoders + .iter() + .filter(|e| !e.starts_with("lib")) + .map(|s| s.as_str()) + .collect(); + + if !hw_encoders.is_empty() { + println!(" ✓ Hardware video encoder ({:?})", hw_encoders); + } else { + println!(" ⚠ No hardware encoders (will use CPU encoding)"); + warnings.push("CPU encoding may impact system performance"); + } + + if diagnostics.d3d11_video_processor_available { + println!(" ✓ D3D11 video processor for frame conversion"); + } else { + println!(" ⚠ D3D11 video processor unavailable (will use CPU conversion)"); + warnings.push("CPU frame conversion may impact performance"); + } + + println!("\n=== Result ==="); + if requirements_met && warnings.is_empty() { + println!("✓ All requirements met - system fully compatible"); + } else if requirements_met { + println!("⚠ Requirements met with warnings:"); + for warning in &warnings { + println!(" - {}", warning); + } + } else { + println!("✗ Missing required components - Cap may not function correctly"); + } + + assert!(requirements_met, "Minimum requirements not met"); +} From e32a49afafdbedc307494ed96e191b26451dd20c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 17:33:17 +0000 Subject: [PATCH 11/26] Use RtlGetVersion for Windows version detection --- crates/scap-direct3d/src/windows_version.rs | 49 +++++++-------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/crates/scap-direct3d/src/windows_version.rs b/crates/scap-direct3d/src/windows_version.rs index 39cdd89c86..100aaa3588 100644 --- a/crates/scap-direct3d/src/windows_version.rs +++ b/crates/scap-direct3d/src/windows_version.rs @@ -1,10 +1,12 @@ #![cfg(windows)] use std::sync::OnceLock; -use windows::Win32::System::SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW}; +use windows::Win32::System::SystemInformation::OSVERSIONINFOEXW; static DETECTED_VERSION: OnceLock> = OnceLock::new(); +type RtlGetVersionFn = unsafe extern "system" fn(*mut OSVERSIONINFOEXW) -> i32; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct WindowsVersion { pub major: u32, @@ -14,7 +16,7 @@ pub struct WindowsVersion { impl WindowsVersion { pub fn detect() -> Option { - *DETECTED_VERSION.get_or_init(|| detect_version_internal()) + *DETECTED_VERSION.get_or_init(detect_version_internal) } pub fn meets_minimum_requirements(&self) -> bool { @@ -45,15 +47,22 @@ impl WindowsVersion { fn detect_version_internal() -> Option { unsafe { + let ntdll = + windows::Win32::System::LibraryLoader::GetModuleHandleW(windows::core::w!("ntdll.dll")) + .ok()?; + + let rtl_get_version: RtlGetVersionFn = + std::mem::transmute(windows::Win32::System::LibraryLoader::GetProcAddress( + ntdll, + windows::core::s!("RtlGetVersion"), + )?); + let mut info = OSVERSIONINFOEXW { dwOSVersionInfoSize: std::mem::size_of::() as u32, ..Default::default() }; - let info_ptr = &mut info as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; - - #[allow(deprecated)] - if GetVersionExW(info_ptr).is_ok() { + if rtl_get_version(&mut info) == 0 { let version = WindowsVersion { major: info.dwMajorVersion, minor: info.dwMinorVersion, @@ -65,37 +74,13 @@ fn detect_version_internal() -> Option { minor = version.minor, build = version.build, display_name = %version.display_name(), - "Detected Windows version" - ); - - return Some(version); - } - - let mut basic_info = OSVERSIONINFOW { - dwOSVersionInfoSize: std::mem::size_of::() as u32, - ..Default::default() - }; - - #[allow(deprecated)] - if GetVersionExW(&mut basic_info).is_ok() { - let version = WindowsVersion { - major: basic_info.dwMajorVersion, - minor: basic_info.dwMinorVersion, - build: basic_info.dwBuildNumber, - }; - - tracing::debug!( - major = version.major, - minor = version.minor, - build = version.build, - display_name = %version.display_name(), - "Detected Windows version (basic)" + "Detected Windows version via RtlGetVersion" ); return Some(version); } - tracing::warn!("Failed to detect Windows version"); + tracing::warn!("RtlGetVersion failed"); None } } From 4973b1b76b94e2e220df4b4b33d3f4f3eac57977 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 17:33:25 +0000 Subject: [PATCH 12/26] Add encoder health monitoring and fallback logic --- .claude/settings.local.json | 3 +- crates/enc-mediafoundation/src/video/h264.rs | 241 ++++++++++++++++-- crates/recording/src/output_pipeline/win.rs | 166 +++++++----- .../src/output_pipeline/win_segmented.rs | 84 +++--- .../output_pipeline/win_segmented_camera.rs | 88 ++++--- crates/scap-direct3d/Cargo.toml | 1 + 6 files changed, 432 insertions(+), 151 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 8e65ddf103..c45c994822 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -44,7 +44,8 @@ "WebFetch(domain:docs.rs)", "WebFetch(domain:gix.github.io)", "Bash(cargo clean:*)", - "Bash(cargo test:*)" + "Bash(cargo test:*)", + "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")" ], "deny": [], "ask": [] diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index dd371989c2..129ba0024f 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -3,9 +3,12 @@ use crate::{ mft::EncoderDevice, video::{NewVideoProcessorError, VideoProcessor}, }; -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, }; use windows::{ Foundation::TimeSpan, @@ -36,6 +39,89 @@ use windows::{ }; const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; +const MAX_INPUT_WITHOUT_OUTPUT: u32 = 30; +const MAX_PROCESS_INPUT_FAILURES: u32 = 5; +const ENCODER_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone)] +pub struct EncoderHealthStatus { + pub inputs_without_output: u32, + pub consecutive_process_failures: u32, + pub total_frames_encoded: u64, + pub is_healthy: bool, + pub failure_reason: Option, +} + +#[derive(Debug, Clone)] +pub enum EncoderFailureReason { + Stalled, + ConsecutiveProcessFailures, + Timeout, + TooManyEmptySamples, +} + +struct EncoderHealthMonitor { + inputs_without_output: u32, + consecutive_process_failures: u32, + total_frames_encoded: u64, + last_output_time: Instant, +} + +impl EncoderHealthMonitor { + fn new() -> Self { + Self { + inputs_without_output: 0, + consecutive_process_failures: 0, + total_frames_encoded: 0, + last_output_time: Instant::now(), + } + } + + fn record_input(&mut self) { + self.inputs_without_output += 1; + } + + fn record_output(&mut self) { + self.inputs_without_output = 0; + self.consecutive_process_failures = 0; + self.total_frames_encoded += 1; + self.last_output_time = Instant::now(); + } + + fn record_process_failure(&mut self) { + self.consecutive_process_failures += 1; + } + + fn reset_process_failures(&mut self) { + self.consecutive_process_failures = 0; + } + + fn check_health(&self) -> EncoderHealthStatus { + let mut is_healthy = true; + let mut failure_reason = None; + + if self.inputs_without_output > MAX_INPUT_WITHOUT_OUTPUT { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::Stalled); + } else if self.consecutive_process_failures >= MAX_PROCESS_INPUT_FAILURES { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::ConsecutiveProcessFailures); + } else if self.last_output_time.elapsed() > ENCODER_OPERATION_TIMEOUT + && self.total_frames_encoded > 0 + { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::Timeout); + } + + EncoderHealthStatus { + inputs_without_output: self.inputs_without_output, + consecutive_process_failures: self.consecutive_process_failures, + total_frames_encoded: self.total_frames_encoded, + is_healthy, + failure_reason, + } + } +} pub struct VideoEncoderOutputSample { sample: IMFSample, @@ -98,6 +184,36 @@ pub enum HandleNeedsInputError { ProcessInput(windows::core::Error), } +#[derive(Clone, Debug, thiserror::Error)] +pub enum EncoderRuntimeError { + #[error("Windows error: {0}")] + Windows(windows::core::Error), + #[error( + "Encoder unhealthy: {reason:?} (inputs_without_output={inputs_without_output}, process_failures={process_failures}, frames_encoded={frames_encoded})" + )] + EncoderUnhealthy { + reason: EncoderFailureReason, + inputs_without_output: u32, + process_failures: u32, + frames_encoded: u64, + }, +} + +impl EncoderRuntimeError { + pub fn should_fallback(&self) -> bool { + match self { + EncoderRuntimeError::Windows(_) => false, + EncoderRuntimeError::EncoderUnhealthy { .. } => true, + } + } +} + +impl From for EncoderRuntimeError { + fn from(err: windows::core::Error) -> Self { + EncoderRuntimeError::Windows(err) + } +} + unsafe impl Send for H264Encoder {} impl H264Encoder { @@ -385,12 +501,35 @@ impl H264Encoder { &self.output_type } + pub fn validate(&self) -> Result<(), NewVideoEncoderError> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + } + Ok(()) + } + pub fn run( &mut self, should_stop: Arc, mut get_frame: impl FnMut() -> windows::core::Result>, mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, - ) -> windows::core::Result<()> { + ) -> Result { + let mut health_monitor = EncoderHealthMonitor::new(); + unsafe { self.transform .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; @@ -399,33 +538,72 @@ impl H264Encoder { self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; - let mut consecutive_empty_samples = 0; + let mut consecutive_empty_samples: u8 = 0; let mut should_exit = false; while !should_exit { + let health_status = health_monitor.check_health(); + if !health_status.is_healthy { + if let Some(reason) = health_status.failure_reason { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status.inputs_without_output, + process_failures: health_status.consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); + } + } + let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; let event_type = MF_EVENT_TYPE(event.GetType()? as i32); match event_type { MediaFoundation::METransformNeedInput => { + health_monitor.record_input(); should_exit = true; if !should_stop.load(Ordering::SeqCst) && let Some((texture, timestamp)) = get_frame()? { - self.video_processor.process_texture(&texture)?; - let input_buffer = { - MFCreateDXGISurfaceBuffer( + let process_result = (|| -> windows::core::Result<()> { + self.video_processor.process_texture(&texture)?; + let input_buffer = MFCreateDXGISurfaceBuffer( &ID3D11Texture2D::IID, self.video_processor.output_texture(), 0, false, - )? - }; - let mf_sample = MFCreateSample()?; - mf_sample.AddBuffer(&input_buffer)?; - mf_sample.SetSampleTime(timestamp.Duration)?; - self.transform - .ProcessInput(self.input_stream_id, &mf_sample, 0)?; - should_exit = false; + )?; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + Ok(()) + })(); + + match process_result { + Ok(()) => { + health_monitor.reset_process_failures(); + should_exit = false; + } + Err(_) => { + health_monitor.record_process_failure(); + let health_status = health_monitor.check_health(); + if !health_status.is_healthy { + if let Some(reason) = health_status.failure_reason { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status + .inputs_without_output, + process_failures: health_status + .consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); + } + } + should_exit = false; + } + } } } MediaFoundation::METransformHaveOutput => { @@ -435,25 +613,24 @@ impl H264Encoder { ..Default::default() }; - // ProcessOutput may succeed but not populate pSample in some edge cases - // (e.g., hardware encoder transient failures, specific MFT implementations). - // This is a known contract violation by certain Media Foundation Transforms. - // We handle this gracefully by skipping the frame instead of panicking. let mut output_buffers = [output_buffer]; self.transform .ProcessOutput(0, &mut output_buffers, &mut status)?; - // Use the sample directly without cloning to prevent memory leaks if let Some(sample) = output_buffers[0].pSample.take() { consecutive_empty_samples = 0; + health_monitor.record_output(); on_sample(sample)?; } else { consecutive_empty_samples += 1; if consecutive_empty_samples > MAX_CONSECUTIVE_EMPTY_SAMPLES { - return Err(windows::core::Error::new( - windows::core::HRESULT(0), - "Too many consecutive empty samples", - )); + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason: EncoderFailureReason::TooManyEmptySamples, + inputs_without_output: health_monitor.inputs_without_output, + process_failures: health_monitor.consecutive_process_failures, + frames_encoded: health_monitor.total_frames_encoded, + }); } } } @@ -471,7 +648,19 @@ impl H264Encoder { .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; } - Ok(()) + Ok(health_monitor.check_health()) + } + + fn cleanup_encoder(&mut self) -> windows::core::Result<()> { + unsafe { + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0); + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + } } } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index b332813b8e..6312f370be 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -189,6 +189,12 @@ impl Muxer for WindowsMuxer { config.bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Hardware encoder validation failed: {e}" + ))); + } + let width = match u32::try_from(output_size.Width) { Ok(width) if width > 0 => width, _ => { @@ -260,36 +266,54 @@ impl Muxer for WindowsMuxer { either::Left((mut encoder, mut muxer)) => { trace!("Running native encoder"); let mut first_timestamp: Option = None; - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = output.lock().unwrap(); + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more frames available"); + return Ok(None); + }; - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + let frame_time = duration_to_timespan(relative); - Ok(()) - }, - ) - .context("run native encoder") + Ok(Some((frame.texture().clone(), frame_time))) + }, + |output_sample| { + let mut output = output.lock().unwrap(); + + let _ = muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| format!("WriteSample: {e}")); + + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + debug!( + "Hardware encoder completed: {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Hardware encoder error: {}", e)) + } + } } either::Right(mut encoder) => { while let Ok(Some((frame, time))) = video_rx.recv() { @@ -571,6 +595,12 @@ impl Muxer for WindowsCameraMuxer { bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Camera hardware encoder validation failed: {e}" + ))); + } + let muxer = { let mut output_guard = match output.lock() { Ok(guard) => guard, @@ -655,43 +685,55 @@ impl Muxer for WindowsCameraMuxer { if let Ok(Some((texture, frame_time))) = process_frame(first_frame.0, first_frame.1) { - encoder - .run( - Arc::new(AtomicBool::default()), - || { - if frame_count > 0 { - let Ok(Some((frame, timestamp))) = video_rx.recv() - else { - trace!("No more camera frames available"); - return Ok(None); - }; - frame_count += 1; - if frame_count.is_multiple_of(30) { - debug!( - "Windows camera encoder: processed {} frames", - frame_count - ); - } - return process_frame(frame, timestamp); - } + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + if frame_count > 0 { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more camera frames available"); + return Ok(None); + }; frame_count += 1; - Ok(Some((texture.clone(), frame_time))) - }, - |output_sample| { - let mut output = output.lock().unwrap(); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); - Ok(()) - }, - ) - .context("run camera encoder")?; + if frame_count.is_multiple_of(30) { + debug!( + "Windows camera encoder: processed {} frames", + frame_count + ); + } + return process_frame(frame, timestamp); + } + frame_count += 1; + Ok(Some((texture.clone(), frame_time))) + }, + |output_sample| { + let mut output = output.lock().unwrap(); + let _ = muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| format!("WriteSample: {e}")); + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + info!( + "Windows camera encoder finished (hardware): {} frames encoded", + health_status.total_frames_encoded + ); + } + Err(e) => { + if e.should_fallback() { + error!( + "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + return Err(anyhow!("Camera hardware encoder error: {}", e)); + } + } } - info!( - "Windows camera encoder finished (hardware): {} frames encoded", - frame_count - ); Ok(()) } either::Right(mut encoder) => { diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs index 7976712d6e..c83e64b735 100644 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ b/crates/recording/src/output_pipeline/win_segmented.rs @@ -424,6 +424,12 @@ impl WindowsSegmentedMuxer { bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Hardware encoder validation failed: {e}" + ))); + } + let width = match u32::try_from(output_size.Width) { Ok(width) if width > 0 => width, _ => { @@ -495,36 +501,54 @@ impl WindowsSegmentedMuxer { either::Left((mut encoder, mut muxer)) => { trace!("Running native encoder for segment"); let mut first_timestamp: Option = None; - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = output_clone.lock().unwrap(); - - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); - - Ok(()) - }, - ) - .context("run native encoder for segment") + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more frames available for segment"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + let frame_time = duration_to_timespan(relative); + + Ok(Some((frame.texture().clone(), frame_time))) + }, + |output_sample| { + let mut output = output_clone.lock().unwrap(); + + let _ = muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| format!("WriteSample: {e}")); + + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + debug!( + "Hardware encoder completed for segment: {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Hardware encoder error: {}", e)) + } + } } either::Right(mut encoder) => { while let Ok(Some((frame, time))) = video_rx.recv() { diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 88462f1d67..36ec47c86f 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -430,6 +430,12 @@ impl WindowsSegmentedCameraMuxer { bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Camera hardware encoder validation failed: {e}" + ))); + } + let muxer = { let mut output_guard = match output_clone.lock() { Ok(guard) => guard, @@ -491,38 +497,56 @@ impl WindowsSegmentedCameraMuxer { let mut first_timestamp: Option = None; - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }, - |output_sample| { - let mut output = output_clone.lock().unwrap(); - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("WriteSample: {e}"), - ) - }) - }, - ) - .context("run camera encoder for segment") + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more camera frames available for segment"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + + let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; + Ok(Some((texture, duration_to_timespan(relative)))) + }, + |output_sample| { + let mut output = output_clone.lock().unwrap(); + muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| { + windows::core::Error::new( + windows::core::HRESULT(-1), + format!("WriteSample: {e}"), + ) + }) + }, + ); + + match result { + Ok(health_status) => { + info!( + "Camera segment encoder finished (hardware): {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Camera hardware encoder error: {}", e)) + } + } } either::Right(mut encoder) => { info!( diff --git a/crates/scap-direct3d/Cargo.toml b/crates/scap-direct3d/Cargo.toml index f95306ec84..928ce647c1 100644 --- a/crates/scap-direct3d/Cargo.toml +++ b/crates/scap-direct3d/Cargo.toml @@ -30,6 +30,7 @@ windows = { workspace = true, features = [ "Storage_Search", "Storage_Streams", "Win32_System_SystemInformation", + "Win32_System_LibraryLoader", ] } [dev-dependencies] From 1784fb37a778188a8a224729023775910a35a7e3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 18:08:43 +0000 Subject: [PATCH 13/26] Enhance GPU diagnostics and optimize UYVY to YUYV conversion --- apps/desktop/src/utils/tauri.ts | 6 +- crates/camera-windows/src/lib.rs | 240 +++++++++++++++++++- crates/frame-converter/src/d3d11.rs | 177 +++++++++++++-- crates/recording/src/diagnostics.rs | 143 +++++++++++- crates/recording/src/output_pipeline/win.rs | 44 +++- 5 files changed, 579 insertions(+), 31 deletions(-) diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 1afc1a0109..f78bc7845a 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -358,6 +358,7 @@ uploadProgressEvent: "upload-progress-event" /** user-defined types **/ +export type AllGpusInfo = { gpus: GpuInfoDiag[]; primaryGpuIndex: number | null; isMultiGpuSystem: boolean; hasDiscreteGpu: boolean } export type Annotation = { id: string; type: AnnotationType; x: number; y: number; width: number; height: number; strokeColor: string; strokeWidth: number; fillColor: string; opacity: number; rotation: number; text: string | null; maskType?: MaskType | null; maskLevel?: number | null } export type AnnotationType = "arrow" | "circle" | "rectangle" | "text" | "mask" export type AppTheme = "system" | "light" | "dark" @@ -423,7 +424,7 @@ quality: number | null; * Whether to prioritize speed over quality (default: false) */ fast: boolean | null } -export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number } +export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number; adapterIndex: number; isSoftwareAdapter: boolean; isBasicRenderDriver: boolean; supportsHardwareEncoding: boolean } export type HapticPattern = "alignment" | "levelChange" | "generic" export type HapticPerformanceTime = "default" | "now" | "drawCompleted" export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; shift: boolean } @@ -478,6 +479,7 @@ export type RecordingStatus = "pending" | "recording" export type RecordingStopped = null export type RecordingTargetMode = "display" | "window" | "area" export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY } +export type RenderingStatus = { isUsingSoftwareRendering: boolean; isUsingBasicRenderDriver: boolean; hardwareEncodingAvailable: boolean; warningMessage: string | null } export type RequestOpenRecordingPicker = { target_mode: RecordingTargetMode | null } export type RequestOpenSettings = { page: string } export type RequestScreenCapturePrewarm = { force?: boolean } @@ -498,7 +500,7 @@ export type StartRecordingInputs = { capture_target: ScreenCaptureTarget; captur export type StereoMode = "stereo" | "monoL" | "monoR" export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments } export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsRemux" } | { status: "Failed"; error: string } | { status: "Complete" } -export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean } +export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; allGpus: AllGpusInfo | null; renderingStatus: RenderingStatus; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean } export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } export type TextSegment = { start: number; end: number; enabled?: boolean; content?: string; center?: XY; size?: XY; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number } export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[] } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index 58aa97349e..e3fac3c88a 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -21,12 +21,148 @@ const MF_VIDEO_FORMAT_P010: GUID = GUID::from_u128(0x30313050_0000_0010_8000_00a const MEDIASUBTYPE_Y800: GUID = GUID::from_u128(0x30303859_0000_0010_8000_00aa00389b71); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeviceCategory { + Physical, + Virtual, + CaptureCard, +} + +impl DeviceCategory { + pub fn is_virtual(&self) -> bool { + matches!(self, DeviceCategory::Virtual) + } + + pub fn is_capture_card(&self) -> bool { + matches!(self, DeviceCategory::CaptureCard) + } +} + +const VIRTUAL_CAMERA_PATTERNS: &[&str] = &[ + "obs", + "virtual", + "snap camera", + "manycam", + "xsplit", + "streamlabs", + "droidcam", + "iriun", + "epoccam", + "ndi", + "newtek", + "camtwist", + "mmhmm", + "chromacam", + "vtuber", + "prism live", + "camo", + "avatarify", + "facerig", +]; + +const CAPTURE_CARD_PATTERNS: &[&str] = &[ + "elgato", + "avermedia", + "magewell", + "blackmagic", + "decklink", + "intensity", + "ultrastudio", + "atomos", + "hauppauge", + "startech", + "j5create", + "razer ripsaw", + "pengo", + "evga xr1", + "nzxt signal", + "genki shadowcast", + "cam link", + "live gamer", + "game capture", +]; + +fn detect_device_category(name: &OsStr, model_id: Option<&str>) -> DeviceCategory { + let name_lower = name.to_string_lossy().to_lowercase(); + let model_lower = model_id.map(|m| m.to_lowercase()); + + let matches_pattern = |patterns: &[&str]| { + patterns.iter().any(|pattern| { + name_lower.contains(pattern) + || model_lower.as_ref().is_some_and(|m| m.contains(pattern)) + }) + }; + + if matches_pattern(CAPTURE_CARD_PATTERNS) { + DeviceCategory::CaptureCard + } else if matches_pattern(VIRTUAL_CAMERA_PATTERNS) { + DeviceCategory::Virtual + } else { + DeviceCategory::Physical + } +} + +#[derive(Debug, Clone)] +pub struct FormatPreference { + pub width: u32, + pub height: u32, + pub frame_rate: f32, + pub format_priority: Vec, +} + +impl FormatPreference { + pub fn new(width: u32, height: u32, frame_rate: f32) -> Self { + Self { + width, + height, + frame_rate, + format_priority: vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::YUV420P, + PixelFormat::MJPEG, + PixelFormat::RGB32, + ], + } + } + + pub fn with_format_priority(mut self, priority: Vec) -> Self { + self.format_priority = priority; + self + } + + pub fn for_hardware_encoding() -> Self { + Self::new(1920, 1080, 30.0).with_format_priority(vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::YUV420P, + ]) + } + + pub fn for_capture_card() -> Self { + Self::new(1920, 1080, 60.0).with_format_priority(vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::P010, + ]) + } +} + +impl Default for FormatPreference { + fn default() -> Self { + Self::new(1920, 1080, 30.0) + } +} + #[derive(Clone)] pub struct VideoDeviceInfo { id: OsString, name: OsString, model_id: Option, - // formats: Vec, + category: DeviceCategory, inner: VideoDeviceInfoInner, } @@ -67,6 +203,104 @@ impl VideoDeviceInfo { self.model_id.as_deref() } + pub fn category(&self) -> DeviceCategory { + self.category + } + + pub fn is_virtual_camera(&self) -> bool { + self.category.is_virtual() + } + + pub fn is_capture_card(&self) -> bool { + self.category.is_capture_card() + } + + pub fn is_high_bandwidth(&self) -> bool { + if !self.is_capture_card() { + return false; + } + self.formats().iter().any(|f| { + let pixels = f.width() as u64 * f.height() as u64; + let fps = f.frame_rate() as u64; + pixels >= 3840 * 2160 && fps >= 30 + }) + } + + pub fn max_resolution(&self) -> Option<(u32, u32)> { + self.formats() + .iter() + .map(|f| (f.width(), f.height())) + .max_by_key(|(w, h)| (*w as u64) * (*h as u64)) + } + + pub fn find_best_format(&self, preference: &FormatPreference) -> Option { + let formats = self.formats(); + if formats.is_empty() { + return None; + } + + let target_pixels = preference.width as u64 * preference.height as u64; + + let score_format = |f: &VideoFormat| { + let format_priority = preference + .format_priority + .iter() + .position(|&pf| pf == f.pixel_format()) + .map(|pos| 1000 - pos as i32) + .unwrap_or(0); + + let pixels = f.width() as u64 * f.height() as u64; + let resolution_score = if pixels == target_pixels { + 500 + } else if pixels > target_pixels { + 400 - ((pixels - target_pixels) / 10000).min(300) as i32 + } else { + 300 - ((target_pixels - pixels) / 10000).min(200) as i32 + }; + + let fps_diff = (f.frame_rate() - preference.frame_rate).abs(); + let fps_score = 100 - (fps_diff * 10.0).min(100.0) as i32; + + format_priority + resolution_score + fps_score + }; + + formats.into_iter().max_by_key(score_format) + } + + pub fn find_format_with_fallback(&self, preference: &FormatPreference) -> Option { + if let Some(format) = self.find_best_format(preference) { + return Some(format); + } + + let fallback_formats = [ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::MJPEG, + PixelFormat::RGB32, + PixelFormat::YUV420P, + ]; + + let formats = self.formats(); + for fallback_pixel_format in fallback_formats { + if let Some(format) = formats + .iter() + .filter(|f| f.pixel_format() == fallback_pixel_format) + .max_by_key(|f| { + let res_score = (f.width() as i32).min(preference.width as i32) + + (f.height() as i32).min(preference.height as i32); + let fps_score = + (100.0 - (f.frame_rate() - preference.frame_rate).abs().min(100.0)) as i32; + res_score + fps_score + }) + { + return Some(format.clone()); + } + } + + formats.into_iter().next() + } + pub fn is_mf(&self) -> bool { matches!(self.inner, VideoDeviceInfoInner::MediaFoundation { .. }) } @@ -285,11 +519,13 @@ pub fn get_devices() -> Result, GetDevicesError> { let name = device.name()?; let id = device.id()?; let model_id = device.model_id(); + let category = detect_device_category(&name, model_id.as_deref()); Ok::<_, windows_core::Error>(VideoDeviceInfo { name, id, model_id, + category, inner: VideoDeviceInfoInner::MediaFoundation { device }, }) }) @@ -308,11 +544,13 @@ pub fn get_devices() -> Result, GetDevicesError> { let id = device.id()?; let name = device.name()?; let model_id = device.model_id(); + let category = detect_device_category(&name, model_id.as_deref()); Some(VideoDeviceInfo { name, id, model_id, + category, inner: VideoDeviceInfoInner::DirectShow(device), }) }) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 34eb84e686..8dac4f6dd6 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -73,6 +73,26 @@ pub struct GpuInfo { pub device_id: u32, pub description: String, pub dedicated_video_memory: u64, + pub adapter_index: u32, + pub is_software_adapter: bool, +} + +impl GpuInfo { + pub fn is_basic_render_driver(&self) -> bool { + self.vendor == GpuVendor::Microsoft + && (self.description.contains("Basic Render Driver") + || self.description.contains("Microsoft Basic") + || self.dedicated_video_memory == 0) + } + + pub fn is_warp(&self) -> bool { + self.description.contains("Microsoft Basic Render Driver") + || self.description.contains("WARP") + } + + pub fn supports_hardware_encoding(&self) -> bool { + !self.is_software_adapter && !self.is_basic_render_driver() + } } impl GpuInfo { @@ -90,19 +110,30 @@ impl GpuInfo { } static DETECTED_GPU: OnceLock> = OnceLock::new(); +static ALL_GPUS: OnceLock> = OnceLock::new(); pub fn detect_primary_gpu() -> Option<&'static GpuInfo> { DETECTED_GPU .get_or_init(|| { - let result = detect_primary_gpu_inner(); + let all_gpus = enumerate_all_gpus(); + let result = select_best_gpu(&all_gpus); if let Some(ref info) = result { tracing::debug!( - "Detected primary GPU: {} (Vendor: {}, VendorID: 0x{:04X}, VRAM: {} MB)", + "Selected primary GPU: {} (Vendor: {}, VendorID: 0x{:04X}, VRAM: {} MB, Adapter: {}, SoftwareRenderer: {})", info.description, info.vendor_name(), info.vendor_id, - info.dedicated_video_memory / (1024 * 1024) + info.dedicated_video_memory / (1024 * 1024), + info.adapter_index, + info.is_software_adapter ); + + if info.is_basic_render_driver() { + tracing::warn!( + "Detected Microsoft Basic Render Driver - hardware encoding will be disabled. \ + This may indicate missing GPU drivers or a remote desktop session." + ); + } } else { tracing::debug!("No GPU detected via DXGI, using default encoder order"); } @@ -111,29 +142,126 @@ pub fn detect_primary_gpu() -> Option<&'static GpuInfo> { .as_ref() } -fn detect_primary_gpu_inner() -> Option { +pub fn get_all_gpus() -> &'static Vec { + ALL_GPUS.get_or_init(enumerate_all_gpus) +} + +fn enumerate_all_gpus() -> Vec { + let mut gpus = Vec::new(); + unsafe { - let factory: IDXGIFactory1 = CreateDXGIFactory1().ok()?; - let adapter: IDXGIAdapter = factory.EnumAdapters(0).ok()?; - let desc = adapter.GetDesc().ok()?; + let factory: IDXGIFactory1 = match CreateDXGIFactory1() { + Ok(f) => f, + Err(e) => { + tracing::warn!("Failed to create DXGI factory: {e:?}"); + return gpus; + } + }; - let description = String::from_utf16_lossy( - &desc - .Description - .iter() - .take_while(|&&c| c != 0) - .copied() - .collect::>(), - ); + let mut adapter_index = 0u32; + loop { + let adapter: IDXGIAdapter = match factory.EnumAdapters(adapter_index) { + Ok(a) => a, + Err(_) => break, + }; - Some(GpuInfo { - vendor: GpuVendor::from_id(desc.VendorId), - vendor_id: desc.VendorId, - device_id: desc.DeviceId, - description, - dedicated_video_memory: desc.DedicatedVideoMemory as u64, + if let Ok(desc) = adapter.GetDesc() { + let description = String::from_utf16_lossy( + &desc + .Description + .iter() + .take_while(|&&c| c != 0) + .copied() + .collect::>(), + ); + + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + + let gpu_info = GpuInfo { + vendor: GpuVendor::from_id(desc.VendorId), + vendor_id: desc.VendorId, + device_id: desc.DeviceId, + description: description.clone(), + dedicated_video_memory: desc.DedicatedVideoMemory as u64, + adapter_index, + is_software_adapter: is_software, + }; + + tracing::debug!( + "Found GPU adapter {}: {} (Vendor: 0x{:04X}, VRAM: {} MB, Software: {})", + adapter_index, + description, + desc.VendorId, + desc.DedicatedVideoMemory / (1024 * 1024), + is_software + ); + + gpus.push(gpu_info); + } + + adapter_index += 1; + } + } + + if gpus.is_empty() { + tracing::warn!("No GPU adapters found via DXGI enumeration"); + } else { + tracing::info!("Enumerated {} GPU adapter(s)", gpus.len()); + } + + gpus +} + +fn select_best_gpu(gpus: &[GpuInfo]) -> Option { + if gpus.is_empty() { + return None; + } + + let hardware_gpus: Vec<&GpuInfo> = gpus.iter().filter(|g| !g.is_software_adapter).collect(); + + if hardware_gpus.is_empty() { + tracing::warn!("No hardware GPUs found, falling back to software adapter"); + return gpus.first().cloned(); + } + + let discrete_gpus: Vec<&GpuInfo> = hardware_gpus + .iter() + .filter(|g| { + matches!( + g.vendor, + GpuVendor::Nvidia | GpuVendor::Amd | GpuVendor::Qualcomm | GpuVendor::Arm + ) }) + .copied() + .collect(); + + if !discrete_gpus.is_empty() { + let best = discrete_gpus + .iter() + .max_by_key(|g| g.dedicated_video_memory) + .unwrap(); + + if discrete_gpus.len() > 1 || hardware_gpus.len() > 1 { + tracing::info!( + "Multi-GPU system detected ({} GPUs). Selected {} with {} MB VRAM for encoding.", + gpus.len(), + best.description, + best.dedicated_video_memory / (1024 * 1024) + ); + } + + return Some((*best).clone()); } + + let best = hardware_gpus + .iter() + .max_by_key(|g| g.dedicated_video_memory) + .unwrap(); + + Some((*best).clone()) } struct D3D11Resources { @@ -189,12 +317,19 @@ fn get_gpu_info(device: &ID3D11Device) -> Result { .collect::>(), ); + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + Ok(GpuInfo { vendor: GpuVendor::from_id(desc.VendorId), vendor_id: desc.VendorId, device_id: desc.DeviceId, description, dedicated_video_memory: desc.DedicatedVideoMemory as u64, + adapter_index: 0, + is_software_adapter: is_software, }) } } diff --git a/crates/recording/src/diagnostics.rs b/crates/recording/src/diagnostics.rs index b304d95d71..3822ca12a9 100644 --- a/crates/recording/src/diagnostics.rs +++ b/crates/recording/src/diagnostics.rs @@ -20,6 +20,28 @@ mod windows_impl { pub vendor: String, pub description: String, pub dedicated_video_memory_mb: f64, + pub adapter_index: u32, + pub is_software_adapter: bool, + pub is_basic_render_driver: bool, + pub supports_hardware_encoding: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct AllGpusInfo { + pub gpus: Vec, + pub primary_gpu_index: Option, + pub is_multi_gpu_system: bool, + pub has_discrete_gpu: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct RenderingStatus { + pub is_using_software_rendering: bool, + pub is_using_basic_render_driver: bool, + pub hardware_encoding_available: bool, + pub warning_message: Option, } #[derive(Debug, Clone, Serialize, Type)] @@ -27,6 +49,8 @@ mod windows_impl { pub struct SystemDiagnostics { pub windows_version: Option, pub gpu_info: Option, + pub all_gpus: Option, + pub rendering_status: RenderingStatus, pub available_encoders: Vec, pub graphics_capture_supported: bool, pub d3d11_video_processor_available: bool, @@ -35,6 +59,8 @@ mod windows_impl { pub fn collect_diagnostics() -> SystemDiagnostics { let windows_version = get_windows_version_info(); let gpu_info = get_gpu_info(); + let all_gpus = get_all_gpus_info(); + let rendering_status = get_rendering_status(&gpu_info); let available_encoders = get_available_encoders(); let graphics_capture_supported = check_graphics_capture_support(); let d3d11_video_processor_available = check_d3d11_video_processor(); @@ -44,7 +70,29 @@ mod windows_impl { tracing::info!(" Windows: {}", ver.display_name); } if let Some(ref gpu) = gpu_info { - tracing::info!(" GPU: {} ({})", gpu.description, gpu.vendor); + tracing::info!( + " Primary GPU: {} ({}) - Software: {}, BasicRender: {}", + gpu.description, + gpu.vendor, + gpu.is_software_adapter, + gpu.is_basic_render_driver + ); + } + if let Some(ref all) = all_gpus { + tracing::info!( + " GPU Count: {}, Multi-GPU: {}, Has Discrete: {}", + all.gpus.len(), + all.is_multi_gpu_system, + all.has_discrete_gpu + ); + } + tracing::info!( + " Rendering: SoftwareRendering={}, HardwareEncoding={}", + rendering_status.is_using_software_rendering, + rendering_status.hardware_encoding_available + ); + if let Some(ref warning) = rendering_status.warning_message { + tracing::warn!(" Warning: {}", warning); } tracing::info!(" Encoders: {:?}", available_encoders); tracing::info!(" Graphics Capture: {}", graphics_capture_supported); @@ -56,6 +104,8 @@ mod windows_impl { SystemDiagnostics { windows_version, gpu_info, + all_gpus, + rendering_status, available_encoders, graphics_capture_supported, d3d11_video_processor_available, @@ -73,14 +123,101 @@ mod windows_impl { }) } - fn get_gpu_info() -> Option { - cap_frame_converter::detect_primary_gpu().map(|info| GpuInfoDiag { + fn gpu_info_to_diag(info: &cap_frame_converter::GpuInfo) -> GpuInfoDiag { + GpuInfoDiag { vendor: info.vendor_name().to_string(), description: info.description.clone(), dedicated_video_memory_mb: (info.dedicated_video_memory / (1024 * 1024)) as f64, + adapter_index: info.adapter_index, + is_software_adapter: info.is_software_adapter, + is_basic_render_driver: info.is_basic_render_driver(), + supports_hardware_encoding: info.supports_hardware_encoding(), + } + } + + fn get_gpu_info() -> Option { + cap_frame_converter::detect_primary_gpu().map(gpu_info_to_diag) + } + + fn get_all_gpus_info() -> Option { + let all_gpus = cap_frame_converter::get_all_gpus(); + + if all_gpus.is_empty() { + return None; + } + + let gpus: Vec = all_gpus.iter().map(gpu_info_to_diag).collect(); + + let primary_gpu = cap_frame_converter::detect_primary_gpu(); + let primary_gpu_index = primary_gpu.and_then(|primary| { + all_gpus + .iter() + .position(|g| g.adapter_index == primary.adapter_index) + .map(|idx| idx as u32) + }); + + let has_discrete = all_gpus.iter().any(|g| { + matches!( + g.vendor, + cap_frame_converter::GpuVendor::Nvidia + | cap_frame_converter::GpuVendor::Amd + | cap_frame_converter::GpuVendor::Qualcomm + | cap_frame_converter::GpuVendor::Arm + ) && !g.is_software_adapter + }); + + Some(AllGpusInfo { + is_multi_gpu_system: gpus.len() > 1, + has_discrete_gpu: has_discrete, + primary_gpu_index, + gpus, }) } + fn get_rendering_status(gpu_info: &Option) -> RenderingStatus { + let (is_software, is_basic_render, hw_encoding, warning) = match gpu_info { + Some(gpu) => { + let is_basic = gpu.is_basic_render_driver; + let is_software = gpu.is_software_adapter; + let hw_available = gpu.supports_hardware_encoding; + + let warning = if is_basic { + Some( + "Microsoft Basic Render Driver detected. This may indicate missing GPU drivers or a remote desktop session. Recording will use software encoding which may impact performance." + .to_string(), + ) + } else if is_software { + Some( + "Software rendering is active. Hardware GPU acceleration is not available. Update your graphics drivers for better performance." + .to_string(), + ) + } else if !hw_available { + Some( + "Hardware encoding may not be available on this GPU. Software encoding will be used as a fallback." + .to_string(), + ) + } else { + None + }; + + (is_software, is_basic, hw_available, warning) + } + None => ( + true, + false, + false, + Some("No GPU detected. Recording will use software encoding.".to_string()), + ), + }; + + RenderingStatus { + is_using_software_rendering: is_software, + is_using_basic_render_driver: is_basic_render, + hardware_encoding_available: hw_encoding, + warning_message: warning, + } + } + fn get_available_encoders() -> Vec { let candidates = [ "h264_nvenc", diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 6312f370be..459ea13e78 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -867,18 +867,54 @@ impl AudioMuxer for WindowsCameraMuxer { fn convert_uyvy_to_yuyv(src: &[u8], width: u32, height: u32) -> Vec { let total_bytes = (width * height * 2) as usize; + let src_len = src.len().min(total_bytes); let mut dst = vec![0u8; total_bytes]; - for i in (0..src.len().min(total_bytes)).step_by(4) { - if i + 3 < src.len() && i + 3 < total_bytes { + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("ssse3") { + unsafe { + convert_uyvy_to_yuyv_ssse3(src, &mut dst, src_len); + } + return dst; + } + } + + convert_uyvy_to_yuyv_scalar(src, &mut dst, src_len); + dst +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "ssse3")] +unsafe fn convert_uyvy_to_yuyv_ssse3(src: &[u8], dst: &mut [u8], len: usize) { + use std::arch::x86_64::*; + + unsafe { + let shuffle_mask = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + + let mut i = 0; + let simd_end = len & !15; + + while i < simd_end { + let chunk = _mm_loadu_si128(src.as_ptr().add(i) as *const __m128i); + let shuffled = _mm_shuffle_epi8(chunk, shuffle_mask); + _mm_storeu_si128(dst.as_mut_ptr().add(i) as *mut __m128i, shuffled); + i += 16; + } + + convert_uyvy_to_yuyv_scalar(&src[i..], &mut dst[i..], len - i); + } +} + +fn convert_uyvy_to_yuyv_scalar(src: &[u8], dst: &mut [u8], len: usize) { + for i in (0..len).step_by(4) { + if i + 3 < src.len() && i + 3 < dst.len() { dst[i] = src[i + 1]; dst[i + 1] = src[i]; dst[i + 2] = src[i + 3]; dst[i + 3] = src[i + 2]; } } - - dst } pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result { From c8bac4f325c44fe13325e2a4d630b35939a75161 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 18:47:58 +0000 Subject: [PATCH 14/26] Improve D3D11 device selection and diagnostics logging --- .claude/settings.local.json | 3 +- crates/frame-converter/src/d3d11.rs | 107 +++++++++++++++++++++++++--- crates/recording/src/diagnostics.rs | 16 ++++- 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index c45c994822..8b8848cd81 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -45,7 +45,8 @@ "WebFetch(domain:gix.github.io)", "Bash(cargo clean:*)", "Bash(cargo test:*)", - "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")" + "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")", + "Bash(cargo build:*)" ], "deny": [], "ask": [] diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 8dac4f6dd6..af77267b31 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -14,7 +14,6 @@ use windows::{ Win32::{ Foundation::{CloseHandle, HANDLE, HMODULE}, Graphics::{ - Direct3D::D3D_DRIVER_TYPE_HARDWARE, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAP_WRITE, @@ -335,17 +334,97 @@ fn get_gpu_info(device: &ID3D11Device) -> Result { } impl D3D11Converter { - pub fn new(config: ConversionConfig) -> Result { - let input_dxgi = pixel_to_dxgi(config.input_format)?; - let output_dxgi = pixel_to_dxgi(config.output_format)?; + fn create_device_on_best_adapter() -> Result<(ID3D11Device, ID3D11DeviceContext), ConvertError> + { + unsafe { + let factory: IDXGIFactory1 = CreateDXGIFactory1().map_err(|e| { + ConvertError::HardwareUnavailable(format!("Failed to create DXGI factory: {e:?}")) + })?; + + let mut best_adapter: Option<(IDXGIAdapter, String, u64)> = None; + let mut adapter_index = 0u32; + + loop { + let adapter: IDXGIAdapter = match factory.EnumAdapters(adapter_index) { + Ok(a) => a, + Err(_) => break, + }; + + if let Ok(desc) = adapter.GetDesc() { + let description = String::from_utf16_lossy( + &desc + .Description + .iter() + .take_while(|&&c| c != 0) + .copied() + .collect::>(), + ); + + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + + if !is_software { + let vendor = GpuVendor::from_id(desc.VendorId); + let is_discrete = matches!( + vendor, + GpuVendor::Nvidia + | GpuVendor::Amd + | GpuVendor::Qualcomm + | GpuVendor::Arm + ); + + let vram = desc.DedicatedVideoMemory as u64; + + let should_use = match &best_adapter { + None => true, + Some((_, _, best_vram)) => { + if is_discrete { + vram > *best_vram + } else { + false + } + } + }; + + if should_use { + tracing::debug!( + "D3D11: Candidate adapter {}: {} (Vendor: 0x{:04X}, VRAM: {} MB, Discrete: {})", + adapter_index, + description, + desc.VendorId, + vram / (1024 * 1024), + is_discrete + ); + best_adapter = Some((adapter, description, vram)); + } + } else { + tracing::debug!( + "D3D11: Skipping software adapter {}: {}", + adapter_index, + description + ); + } + } + + adapter_index += 1; + } + + let (adapter, adapter_name, _) = best_adapter.ok_or_else(|| { + ConvertError::HardwareUnavailable( + "No suitable hardware GPU adapter found".to_string(), + ) + })?; + + tracing::info!("D3D11: Creating device on adapter: {}", adapter_name); - let (device, context) = unsafe { let mut device = None; let mut context = None; D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, + Some(&adapter), + windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN, HMODULE::default(), D3D11_CREATE_DEVICE_VIDEO_SUPPORT, None, @@ -356,7 +435,8 @@ impl D3D11Converter { ) .map_err(|e| { ConvertError::HardwareUnavailable(format!( - "D3D11CreateDevice failed (no hardware GPU available?): {e:?}" + "D3D11CreateDevice failed on {}: {e:?}", + adapter_name )) })?; @@ -367,8 +447,15 @@ impl D3D11Converter { ConvertError::HardwareUnavailable("D3D11 context was null".to_string()) })?; - (device, context) - }; + Ok((device, context)) + } + } + + pub fn new(config: ConversionConfig) -> Result { + let input_dxgi = pixel_to_dxgi(config.input_format)?; + let output_dxgi = pixel_to_dxgi(config.output_format)?; + + let (device, context) = Self::create_device_on_best_adapter()?; let gpu_info = get_gpu_info(&device)?; diff --git a/crates/recording/src/diagnostics.rs b/crates/recording/src/diagnostics.rs index 3822ca12a9..ceee01dd10 100644 --- a/crates/recording/src/diagnostics.rs +++ b/crates/recording/src/diagnostics.rs @@ -53,6 +53,7 @@ mod windows_impl { pub rendering_status: RenderingStatus, pub available_encoders: Vec, pub graphics_capture_supported: bool, + #[serde(rename = "d3D11VideoProcessorAvailable")] pub d3d11_video_processor_available: bool, } @@ -255,7 +256,20 @@ mod windows_impl { 1080, ); - cap_frame_converter::D3D11Converter::new(test_config).is_ok() + match cap_frame_converter::D3D11Converter::new(test_config) { + Ok(converter) => { + tracing::debug!( + "D3D11 video processor check passed: {} ({})", + converter.gpu_info().description, + converter.gpu_info().vendor_name() + ); + true + } + Err(e) => { + tracing::warn!("D3D11 video processor check failed: {e:?}"); + false + } + } } } From bdd59a13ba26b510ebf1ea7408e42bcee7153278 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 18:48:05 +0000 Subject: [PATCH 15/26] Set fixed window sizes and enforce on Windows --- apps/desktop/src-tauri/src/windows.rs | 123 +++++++++++++++++++------- 1 file changed, 93 insertions(+), 30 deletions(-) diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index b36d554334..a223e0f721 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -272,15 +272,28 @@ impl ShowCapWindow { let monitor = app.primary_monitor()?.unwrap(); let window = match self { - Self::Setup => self - .window_builder(app, "/setup") - .resizable(false) - .maximized(false) - .center() - .focused(true) - .maximizable(false) - .shadow(true) - .build()?, + Self::Setup => { + let window = self + .window_builder(app, "/setup") + .inner_size(600.0, 600.0) + .min_inner_size(600.0, 600.0) + .resizable(false) + .maximized(false) + .center() + .focused(true) + .maximizable(false) + .shadow(true) + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(600.0, 600.0)); + let _ = window.center(); + } + + window + } Self::Main { init_target_mode } => { if !permissions::do_permissions_check(false).necessary_granted() { return Box::pin(Self::Setup.show(app)).await; @@ -417,7 +430,6 @@ impl ShowCapWindow { window } Self::Settings { page } => { - // Hide main window and target select overlays when settings window opens for (label, window) in app.webview_windows() { if let Ok(id) = CapWindowId::from_str(&label) && matches!( @@ -431,14 +443,26 @@ impl ShowCapWindow { } } - self.window_builder( - app, - format!("/settings/{}", page.clone().unwrap_or_default()), - ) - .resizable(true) - .maximized(false) - .center() - .build()? + let window = self + .window_builder( + app, + format!("/settings/{}", page.clone().unwrap_or_default()), + ) + .inner_size(600.0, 465.0) + .min_inner_size(600.0, 465.0) + .resizable(true) + .maximized(false) + .center() + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(600.0, 465.0)); + let _ = window.center(); + } + + window } Self::Editor { .. } => { if let Some(main) = CapWindowId::Main.get(app) { @@ -448,11 +472,22 @@ impl ShowCapWindow { let _ = camera.close(); }; - self.window_builder(app, "/editor") + let window = self + .window_builder(app, "/editor") .maximizable(true) - .inner_size(1240.0, 800.0) + .inner_size(1275.0, 800.0) + .min_inner_size(1275.0, 800.0) .center() - .build()? + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(1275.0, 800.0)); + let _ = window.center(); + } + + window } Self::ScreenshotEditor { path: _ } => { if let Some(main) = CapWindowId::Main.get(app) { @@ -462,35 +497,55 @@ impl ShowCapWindow { let _ = camera.close(); }; - self.window_builder(app, "/screenshot-editor") + let window = self + .window_builder(app, "/screenshot-editor") .maximizable(true) .inner_size(1240.0, 800.0) + .min_inner_size(800.0, 600.0) .center() - .build()? + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(1240.0, 800.0)); + let _ = window.center(); + } + + window } Self::Upgrade => { - // Hide main window when upgrade window opens if let Some(main) = CapWindowId::Main.get(app) { let _ = main.hide(); } - let mut builder = self + let window = self .window_builder(app, "/upgrade") + .inner_size(950.0, 850.0) + .min_inner_size(950.0, 850.0) .resizable(false) .focused(true) .always_on_top(true) .maximized(false) .shadow(true) - .center(); + .center() + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(950.0, 850.0)); + let _ = window.center(); + } - builder.build()? + window } Self::ModeSelect => { if let Some(main) = CapWindowId::Main.get(app) { let _ = main.hide(); } - let mut builder = self + let window = self .window_builder(app, "/mode-select") .inner_size(580.0, 340.0) .min_inner_size(580.0, 340.0) @@ -499,9 +554,17 @@ impl ShowCapWindow { .maximizable(false) .center() .focused(true) - .shadow(true); + .shadow(true) + .build()?; - builder.build()? + #[cfg(windows)] + { + use tauri::LogicalSize; + let _ = window.set_size(LogicalSize::new(580.0, 340.0)); + let _ = window.center(); + } + + window } Self::Camera => { const WINDOW_SIZE: f64 = 230.0 * 2.0; From b4fcdd5b19e942b655cabb29f53d92dc2c933a10 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:00:26 +0000 Subject: [PATCH 16/26] Add decoder type/status reporting and health monitoring --- crates/rendering/src/decoder/avassetreader.rs | 18 +- crates/rendering/src/decoder/ffmpeg.rs | 56 ++-- .../rendering/src/decoder/media_foundation.rs | 143 ++++++++++- crates/rendering/src/decoder/mod.rs | 240 +++++++++++++++--- crates/rendering/src/lib.rs | 2 +- crates/video-decode/src/ffmpeg.rs | 4 + 6 files changed, 390 insertions(+), 73 deletions(-) diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 8192e0c2ce..408490097b 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -18,7 +18,7 @@ use tokio::{runtime::Handle as TokioHandle, sync::oneshot}; use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; -use super::{FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; +use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; #[derive(Clone)] struct ProcessedFrame { @@ -253,7 +253,7 @@ impl AVAssetReaderDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) { let handle = tokio::runtime::Handle::current(); @@ -265,14 +265,11 @@ impl AVAssetReaderDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, tokio_handle: tokio::runtime::Handle, ) { let mut this = match AVAssetReaderDecoder::new(path, tokio_handle) { - Ok(v) => { - ready_tx.send(Ok(())).ok(); - v - } + Ok(v) => v, Err(e) => { ready_tx.send(Err(e)).ok(); return; @@ -282,6 +279,13 @@ impl AVAssetReaderDecoder { let video_width = this.inner.width(); let video_height = this.inner.height(); + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type: DecoderType::AVAssetReader, + }; + ready_tx.send(Ok(init_result)).ok(); + let mut cache = BTreeMap::::new(); #[allow(unused)] diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 450e8db077..8504f14b88 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -10,10 +10,14 @@ use std::{ sync::{Arc, mpsc}, }; use tokio::sync::oneshot; +use tracing::info; use crate::{DecodedFrame, PixelFormat}; -use super::{FRAME_CACHE_SIZE, VideoDecoderMessage, frame_converter::FrameConverter, pts_to_frame}; +use super::{ + DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, + frame_converter::FrameConverter, pts_to_frame, +}; #[derive(Clone)] struct ProcessedFrame { @@ -134,29 +138,36 @@ pub struct FfmpegDecoder; impl FfmpegDecoder { pub fn spawn( - _name: &'static str, + name: &'static str, path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) -> Result<(), String> { - let (continue_tx, continue_rx) = mpsc::channel(); + let (continue_tx, continue_rx) = mpsc::channel::>(); std::thread::spawn(move || { - let mut this = match cap_video_decode::FFmpegDecoder::new( - path, - Some(if cfg!(target_os = "macos") { - AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX - } else { - AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2 - }), - ) { + let hw_device_type = if cfg!(target_os = "macos") { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX) + } else { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2) + }; + + let mut this = match cap_video_decode::FFmpegDecoder::new(path.clone(), hw_device_type) + { Err(e) => { let _ = continue_tx.send(Err(e)); return; } Ok(v) => { - let _ = continue_tx.send(Ok(())); + let is_hw = v.is_hardware_accelerated(); + let width = v.decoder().width(); + let height = v.decoder().height(); + info!( + "FFmpeg decoder created for '{}': {}x{}, hw_accel={}", + name, width, height, is_hw + ); + let _ = continue_tx.send(Ok((width, height, is_hw))); v } }; @@ -165,10 +176,9 @@ impl FfmpegDecoder { let start_time = this.start_time(); let video_width = this.decoder().width(); let video_height = this.decoder().height(); + let is_hw = this.is_hardware_accelerated(); let mut cache = BTreeMap::::new(); - // active frame is a frame that triggered decode. - // frames that are within render_more_margin of this frame won't trigger decode. #[allow(unused)] let mut last_active_frame = None::; @@ -177,7 +187,17 @@ impl FfmpegDecoder { let mut frames = this.frames(); let mut converter = FrameConverter::new(); - let _ = ready_tx.send(Ok(())); + let decoder_type = if is_hw { + DecoderType::FFmpegHardware + } else { + DecoderType::FFmpegSoftware + }; + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type, + }; + let _ = ready_tx.send(Ok(init_result)); while let Ok(r) = rx.recv() { match r { @@ -356,9 +376,7 @@ impl FfmpegDecoder { } }); - continue_rx.recv().map_err(|e| e.to_string())??; - - Ok(()) + continue_rx.recv().map_err(|e| e.to_string())?.map(|_| ()) } } diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs index 2963f09c04..18a142b734 100644 --- a/crates/rendering/src/decoder/media_foundation.rs +++ b/crates/rendering/src/decoder/media_foundation.rs @@ -2,12 +2,88 @@ use std::{ collections::BTreeMap, path::PathBuf, sync::{Arc, mpsc}, + time::{Duration, Instant}, }; use tokio::sync::oneshot; use tracing::{debug, info, warn}; use windows::Win32::{Foundation::HANDLE, Graphics::Direct3D11::ID3D11Texture2D}; -use super::{DecodedFrame, FRAME_CACHE_SIZE, VideoDecoderMessage}; +use super::{DecodedFrame, DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage}; + +struct DecoderHealthMonitor { + consecutive_errors: u32, + consecutive_texture_read_failures: u32, + total_frames_decoded: u64, + total_errors: u64, + last_successful_decode: Instant, + frame_decode_times: [Duration; 32], + frame_decode_index: usize, + slow_frame_count: u32, +} + +impl DecoderHealthMonitor { + fn new() -> Self { + Self { + consecutive_errors: 0, + consecutive_texture_read_failures: 0, + total_frames_decoded: 0, + total_errors: 0, + last_successful_decode: Instant::now(), + frame_decode_times: [Duration::ZERO; 32], + frame_decode_index: 0, + slow_frame_count: 0, + } + } + + fn record_success(&mut self, decode_time: Duration) { + self.consecutive_errors = 0; + self.consecutive_texture_read_failures = 0; + self.total_frames_decoded += 1; + self.last_successful_decode = Instant::now(); + + self.frame_decode_times[self.frame_decode_index] = decode_time; + self.frame_decode_index = (self.frame_decode_index + 1) % 32; + + const SLOW_FRAME_THRESHOLD: Duration = Duration::from_millis(100); + if decode_time > SLOW_FRAME_THRESHOLD { + self.slow_frame_count += 1; + } + } + + fn record_error(&mut self) { + self.consecutive_errors += 1; + self.total_errors += 1; + } + + fn record_texture_read_failure(&mut self) { + self.consecutive_texture_read_failures += 1; + } + + fn is_healthy(&self) -> bool { + const MAX_CONSECUTIVE_ERRORS: u32 = 10; + const MAX_CONSECUTIVE_TEXTURE_FAILURES: u32 = 5; + const MAX_TIME_SINCE_SUCCESS: Duration = Duration::from_secs(5); + + self.consecutive_errors < MAX_CONSECUTIVE_ERRORS + && self.consecutive_texture_read_failures < MAX_CONSECUTIVE_TEXTURE_FAILURES + && self.last_successful_decode.elapsed() < MAX_TIME_SINCE_SUCCESS + } + + #[allow(dead_code)] + fn average_decode_time(&self) -> Duration { + let sum: Duration = self.frame_decode_times.iter().sum(); + sum / 32 + } + + #[allow(dead_code)] + fn get_health_summary(&self) -> (u64, u64, u32) { + ( + self.total_frames_decoded, + self.total_errors, + self.slow_frame_count, + ) + } +} #[derive(Clone)] struct CachedFrame { @@ -49,7 +125,7 @@ impl MFDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) -> Result<(), String> { let (continue_tx, continue_rx) = mpsc::channel(); @@ -60,14 +136,34 @@ impl MFDecoder { return; } Ok(v) => { + let width = v.width(); + let height = v.height(); + let caps = v.capabilities(); + + let exceeds_hw_limits = width > caps.max_width || height > caps.max_height; + + if exceeds_hw_limits { + warn!( + "Video '{}' dimensions {}x{} exceed hardware decoder limits ({}x{})", + name, width, height, caps.max_width, caps.max_height + ); + let _ = continue_tx.send(Err(format!( + "Video dimensions {}x{} exceed hardware decoder limits {}x{}", + width, height, caps.max_width, caps.max_height + ))); + return; + } + info!( - "MediaFoundation decoder created for '{}': {}x{} @ {:?}fps", + "MediaFoundation decoder created for '{}': {}x{} @ {:?}fps (hw max: {}x{})", name, - v.width(), - v.height(), - v.frame_rate() + width, + height, + v.frame_rate(), + caps.max_width, + caps.max_height ); - let _ = continue_tx.send(Ok(())); + let _ = continue_tx.send(Ok((width, height))); v } }; @@ -77,8 +173,14 @@ impl MFDecoder { let mut cache = BTreeMap::::new(); let mut last_decoded_frame: Option = None; + let mut health = DecoderHealthMonitor::new(); - let _ = ready_tx.send(Ok(())); + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type: DecoderType::MediaFoundation, + }; + let _ = ready_tx.send(Ok(init_result)); while let Ok(r) = rx.recv() { match r { @@ -87,6 +189,16 @@ impl MFDecoder { continue; } + if !health.is_healthy() { + warn!( + name = name, + consecutive_errors = health.consecutive_errors, + texture_failures = health.consecutive_texture_read_failures, + total_decoded = health.total_frames_decoded, + "MediaFoundation decoder unhealthy, performance may degrade" + ); + } + let requested_frame = (requested_time * fps as f32).floor() as u32; if let Some(cached) = cache.get(&requested_frame) { @@ -123,8 +235,10 @@ impl MFDecoder { let mut last_valid_frame: Option = None; loop { + let decode_start = Instant::now(); match decoder.read_sample() { Ok(Some(mf_frame)) => { + let decode_time = decode_start.elapsed(); let frame_number = pts_100ns_to_frame(mf_frame.pts, fps); let nv12_data = match decoder.read_texture_to_cpu( @@ -133,6 +247,7 @@ impl MFDecoder { mf_frame.height, ) { Ok(data) => { + health.record_success(decode_time); debug!( frame = frame_number, data_len = data.data.len(), @@ -140,11 +255,13 @@ impl MFDecoder { uv_stride = data.uv_stride, width = mf_frame.width, height = mf_frame.height, + decode_ms = decode_time.as_millis(), "read_texture_to_cpu succeeded" ); Some(Arc::new(data)) } Err(e) => { + health.record_texture_read_failure(); warn!( "Failed to read texture to CPU for frame {frame_number}: {e}" ); @@ -211,7 +328,11 @@ impl MFDecoder { break; } Err(e) => { - warn!("MediaFoundation read_sample error: {e}"); + health.record_error(); + warn!( + consecutive_errors = health.consecutive_errors, + "MediaFoundation read_sample error: {e}" + ); break; } } @@ -243,9 +364,7 @@ impl MFDecoder { } }); - continue_rx.recv().map_err(|e| e.to_string())??; - - Ok(()) + continue_rx.recv().map_err(|e| e.to_string())?.map(|_| ()) } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 765f0d45ad..4ba74a146c 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -3,9 +3,10 @@ use std::{ fmt, path::PathBuf, sync::{Arc, mpsc}, + time::Duration, }; use tokio::sync::oneshot; -use tracing::debug; +use tracing::{debug, info, warn}; #[cfg(target_os = "macos")] mod avassetreader; @@ -14,6 +15,57 @@ mod frame_converter; #[cfg(target_os = "windows")] mod media_foundation; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecoderType { + #[cfg(target_os = "macos")] + AVAssetReader, + #[cfg(target_os = "windows")] + MediaFoundation, + FFmpegHardware, + FFmpegSoftware, +} + +impl DecoderType { + pub fn is_hardware_accelerated(&self) -> bool { + match self { + #[cfg(target_os = "macos")] + DecoderType::AVAssetReader => true, + #[cfg(target_os = "windows")] + DecoderType::MediaFoundation => true, + DecoderType::FFmpegHardware => true, + DecoderType::FFmpegSoftware => false, + } + } +} + +impl fmt::Display for DecoderType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + #[cfg(target_os = "macos")] + DecoderType::AVAssetReader => write!(f, "AVAssetReader (hardware)"), + #[cfg(target_os = "windows")] + DecoderType::MediaFoundation => write!(f, "MediaFoundation (hardware)"), + DecoderType::FFmpegHardware => write!(f, "FFmpeg (hardware)"), + DecoderType::FFmpegSoftware => write!(f, "FFmpeg (software)"), + } + } +} + +#[derive(Debug, Clone)] +pub struct DecoderStatus { + pub decoder_type: DecoderType, + pub video_width: u32, + pub video_height: u32, + pub fallback_reason: Option, +} + +#[derive(Debug, Clone)] +pub struct DecoderInitResult { + pub width: u32, + pub height: u32, + pub decoder_type: DecoderType, +} + #[cfg(target_os = "macos")] use cidre::{arc::R, cv}; @@ -403,6 +455,7 @@ pub const FRAME_CACHE_SIZE: usize = 750; pub struct AsyncVideoDecoderHandle { sender: mpsc::Sender, offset: f64, + status: DecoderStatus, } impl AsyncVideoDecoderHandle { @@ -425,6 +478,26 @@ impl AsyncVideoDecoderHandle { pub fn get_time(&self, time: f32) -> f32 { time + self.offset as f32 } + + pub fn decoder_status(&self) -> &DecoderStatus { + &self.status + } + + pub fn decoder_type(&self) -> DecoderType { + self.status.decoder_type + } + + pub fn is_hardware_accelerated(&self) -> bool { + self.status.decoder_type.is_hardware_accelerated() + } + + pub fn video_dimensions(&self) -> (u32, u32) { + (self.status.video_width, self.status.video_height) + } + + pub fn fallback_reason(&self) -> Option<&str> { + self.status.fallback_reason.as_deref() + } } pub async fn spawn_decoder( @@ -433,61 +506,160 @@ pub async fn spawn_decoder( fps: u32, offset: f64, ) -> Result { - let (ready_tx, ready_rx) = oneshot::channel::>(); - let (tx, rx) = mpsc::channel(); - - let handle = AsyncVideoDecoderHandle { sender: tx, offset }; - let path_display = path.display().to_string(); + let timeout_duration = Duration::from_secs(30); #[cfg(target_os = "macos")] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + avassetreader::AVAssetReaderDecoder::spawn(name, path, fps, rx, ready_tx); + + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) + } + Ok(Ok(Err(e))) => Err(format!("'{name}' decoder initialization failed: {e}")), + Ok(Err(e)) => Err(format!("'{name}' decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' decoder timed out after 30s initializing: {path_display}" + )), + } } #[cfg(target_os = "windows")] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + match media_foundation::MFDecoder::spawn(name, path.clone(), fps, rx, ready_tx) { - Ok(()) => { - debug!("Using MediaFoundation decoder for '{name}'"); - } + Ok(()) => match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + return Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }); + } + Ok(Ok(Err(e))) => { + warn!( + "MediaFoundation decoder ready but failed for '{}': {}, falling back to FFmpeg", + name, e + ); + } + Ok(Err(e)) => { + warn!( + "MediaFoundation decoder channel closed for '{}': {}, falling back to FFmpeg", + name, e + ); + } + Err(_) => { + warn!( + "MediaFoundation decoder timed out for '{}', falling back to FFmpeg", + name + ); + } + }, Err(mf_err) => { debug!( - "MediaFoundation decoder failed for '{name}': {mf_err}, falling back to FFmpeg" + "MediaFoundation decoder spawn failed for '{}': {}, falling back to FFmpeg", + name, mf_err ); - let (ready_tx, ready_rx_new) = oneshot::channel::>(); - let (tx, rx) = mpsc::channel(); - let handle = AsyncVideoDecoderHandle { sender: tx, offset }; - - ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) - .map_err(|e| format!("'{name}' decoder / {e}"))?; - - return match tokio::time::timeout(std::time::Duration::from_secs(30), ready_rx_new) - .await - { - Ok(result) => result - .map_err(|e| format!("'{name}' decoder channel closed: {e}"))? - .map(|()| handle), - Err(_) => Err(format!( - "'{name}' decoder timed out after 30s initializing: {path_display}" - )), + } + } + + let fallback_reason = + format!("MediaFoundation decoder unavailable for '{name}', using FFmpeg fallback"); + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + + ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) + .map_err(|e| format!("'{name}' FFmpeg fallback decoder / {e}"))?; + + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{}) [fallback]", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: Some(fallback_reason), }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) } + Ok(Ok(Err(e))) => Err(format!( + "'{name}' FFmpeg decoder initialization failed: {e}" + )), + Ok(Err(e)) => Err(format!("'{name}' FFmpeg decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' FFmpeg decoder timed out after 30s initializing: {path_display}" + )), } } #[cfg(not(any(target_os = "macos", target_os = "windows")))] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) .map_err(|e| format!("'{name}' decoder / {e}"))?; - } - match tokio::time::timeout(std::time::Duration::from_secs(30), ready_rx).await { - Ok(result) => result - .map_err(|e| format!("'{name}' decoder channel closed: {e}"))? - .map(|()| handle), - Err(_) => Err(format!( - "'{name}' decoder timed out after 30s initializing: {path_display}" - )), + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) + } + Ok(Ok(Err(e))) => Err(format!("'{name}' decoder initialization failed: {e}")), + Ok(Err(e)) => Err(format!("'{name}' decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' decoder timed out after 30s initializing: {path_display}" + )), + } } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index b781874bf5..4da4bc794f 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -40,7 +40,7 @@ pub mod yuv_converter; mod zoom; pub use coord::*; -pub use decoder::{DecodedFrame, PixelFormat}; +pub use decoder::{DecodedFrame, DecoderStatus, DecoderType, PixelFormat}; pub use frame_pipeline::RenderedFrame; pub use project_recordings::{ProjectRecordingsMeta, SegmentRecordings, Video}; diff --git a/crates/video-decode/src/ffmpeg.rs b/crates/video-decode/src/ffmpeg.rs index 354c71e935..7a7a69f296 100644 --- a/crates/video-decode/src/ffmpeg.rs +++ b/crates/video-decode/src/ffmpeg.rs @@ -281,6 +281,10 @@ impl FFmpegDecoder { pub fn start_time(&self) -> i64 { self.start_time } + + pub fn is_hardware_accelerated(&self) -> bool { + self.hw_device.is_some() + } } unsafe impl Send for FFmpegDecoder {} From 8a33adee4f5b8fc2738f44236079f237a8335394 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:07:16 +0000 Subject: [PATCH 17/26] Enable crash recovery recording by default --- apps/desktop/src-tauri/src/general_settings.rs | 4 ++-- .../src/routes/(window-chrome)/settings/experimental.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 4c1373d43c..f5169a7a27 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -124,7 +124,7 @@ pub struct GeneralSettingsStore { pub instant_mode_max_resolution: u32, #[serde(default)] pub default_project_name_template: Option, - #[serde(default)] + #[serde(default = "default_true")] pub crash_recovery_recording: bool, } @@ -192,7 +192,7 @@ impl Default for GeneralSettingsStore { delete_instant_recordings_after_upload: false, instant_mode_max_resolution: 1920, default_project_name_template: None, - crash_recovery_recording: false, + crash_recovery_recording: true, } } } diff --git a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx index 0e6ba2a7de..3819fdca2b 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx @@ -27,7 +27,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { enableNewRecordingFlow: true, autoZoomOnClicks: false, custom_cursor_capture2: true, - crashRecoveryRecording: false, + crashRecoveryRecording: true, }, ); From 3c8e4a36fdb3c52f99201143d39bccbe5c72da23 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:26:09 +0000 Subject: [PATCH 18/26] clippy --- crates/camera-directshow/examples/cli.rs | 4 +- crates/camera-windows/examples/cli.rs | 1 + crates/cursor-info/examples/cli.rs | 11 +++-- crates/enc-mediafoundation/examples/cli.rs | 16 +++---- crates/frame-converter/src/d3d11.rs | 3 +- crates/recording/examples/camera-benchmark.rs | 28 ++++++------ .../recording/examples/encoding-benchmark.rs | 4 +- .../recording/examples/recording-benchmark.rs | 2 +- crates/recording/tests/hardware_compat.rs | 43 +++++++++---------- crates/rendering/src/cpu_yuv.rs | 18 ++------ crates/scap-direct3d/examples/cli.rs | 5 +-- 11 files changed, 57 insertions(+), 78 deletions(-) diff --git a/crates/camera-directshow/examples/cli.rs b/crates/camera-directshow/examples/cli.rs index f154aa99a6..00161139dd 100644 --- a/crates/camera-directshow/examples/cli.rs +++ b/crates/camera-directshow/examples/cli.rs @@ -56,7 +56,7 @@ mod windows { return None; } - let video_info = &*media_type.video_info(); + let video_info = media_type.video_info(); let width = video_info.bmiHeader.biWidth; let height = video_info.bmiHeader.biHeight; @@ -114,7 +114,7 @@ mod windows { .start_capturing( &selected_format.media_type, Box::new(|frame| { - let data_length = unsafe { frame.sample.GetActualDataLength() }; + let data_length = frame.sample.GetActualDataLength(); println!( "Frame: data_length={data_length:?}, timestamp={:?}", frame.timestamp diff --git a/crates/camera-windows/examples/cli.rs b/crates/camera-windows/examples/cli.rs index fbf898bc06..4725556844 100644 --- a/crates/camera-windows/examples/cli.rs +++ b/crates/camera-windows/examples/cli.rs @@ -69,5 +69,6 @@ mod windows { } } + #[allow(dead_code)] pub struct FormatSelection(VideoFormat); } diff --git a/crates/cursor-info/examples/cli.rs b/crates/cursor-info/examples/cli.rs index 7bb589fab2..cee2b8fdf9 100644 --- a/crates/cursor-info/examples/cli.rs +++ b/crates/cursor-info/examples/cli.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; - -use cap_cursor_info::{CursorShape, CursorShapeMacOS}; +use cap_cursor_info::CursorShape; +#[cfg(target_os = "macos")] +use cap_cursor_info::CursorShapeMacOS; +#[cfg(target_os = "macos")] use sha2::{Digest, Sha256}; +#[cfg(target_os = "macos")] +use std::collections::HashMap; #[allow(unreachable_code)] fn main() { @@ -103,7 +106,7 @@ fn run() { // Try to convert HCURSOR to CursorShape using the TryFrom implementation match CursorShape::try_from(&cursor_info.hCursor) { Ok(cursor_shape) => { - println!("CursorShape: {}", cursor_shape); + println!("CursorShape: {cursor_shape}"); } Err(_) => { println!("Unknown cursor: {:?}", cursor_info.hCursor); diff --git a/crates/enc-mediafoundation/examples/cli.rs b/crates/enc-mediafoundation/examples/cli.rs index f24f0a4fab..458bfb8cc6 100644 --- a/crates/enc-mediafoundation/examples/cli.rs +++ b/crates/enc-mediafoundation/examples/cli.rs @@ -18,7 +18,7 @@ mod win { Foundation::{Metadata::ApiInformation, TimeSpan}, Graphics::Capture::GraphicsCaptureSession, Win32::{ - Media::MediaFoundation::{self, MFSTARTUP_FULL, MFStartup}, + Media::MediaFoundation::{MFSTARTUP_FULL, MFStartup}, System::{ Diagnostics::Debug::{DebugBreak, IsDebuggerPresent}, Threading::GetCurrentProcessId, @@ -44,7 +44,7 @@ mod win { if wait_for_debugger { let pid = unsafe { GetCurrentProcessId() }; - println!("Waiting for a debugger to attach (PID: {})...", pid); + println!("Waiting for a debugger to attach (PID: {pid})..."); loop { if unsafe { IsDebuggerPresent().into() } { break; @@ -64,10 +64,7 @@ mod win { } if verbose { - println!( - "Using index \"{}\" and path \"{}\".", - display_index, output_path - ); + println!("Using index \"{display_index}\" and path \"{output_path}\"."); } let item = Display::primary() @@ -77,7 +74,7 @@ mod win { // Resolve encoding settings let resolution = item.Size()?; - let bit_rate = bit_rate * 1000000; + let _bit_rate = bit_rate * 1000000; // Start the recording { @@ -125,7 +122,7 @@ mod win { ) .unwrap(); - let output_path = std::env::current_dir().unwrap().join(output_path); + let _output_path = std::env::current_dir().unwrap().join(output_path); // let sample_writer = Arc::new(SampleWriter::new(output_path.as_path())?); @@ -216,7 +213,7 @@ mod win { } fn exit_with_error(message: &str) -> ! { - println!("{}", message); + println!("{message}"); std::process::exit(1); } @@ -289,6 +286,7 @@ mod win { } } + #[allow(dead_code)] mod hotkey { use std::sync::atomic::{AtomicI32, Ordering}; use windows::{ diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index af77267b31..36e69371e8 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -435,8 +435,7 @@ impl D3D11Converter { ) .map_err(|e| { ConvertError::HardwareUnavailable(format!( - "D3D11CreateDevice failed on {}: {e:?}", - adapter_name + "D3D11CreateDevice failed on {adapter_name}: {e:?}" )) })?; diff --git a/crates/recording/examples/camera-benchmark.rs b/crates/recording/examples/camera-benchmark.rs index 05f00ff419..5ad07ad524 100644 --- a/crates/recording/examples/camera-benchmark.rs +++ b/crates/recording/examples/camera-benchmark.rs @@ -127,10 +127,7 @@ async fn run_camera_encoding_benchmark( let width = first_frame.inner.width(); let height = first_frame.inner.height(); - println!( - "\nCamera frame format: {:?} {}x{}", - input_format, width, height - ); + println!("\nCamera frame format: {input_format:?} {width}x{height}"); let output_format = Pixel::NV12; let needs_conversion = input_format != output_format; @@ -295,13 +292,12 @@ async fn run_camera_encoding_benchmark( let encode_start = Instant::now(); let timestamp = Duration::from_micros(converted.frame.pts().unwrap_or(0) as u64); - match encoder.queue_preconverted_frame(converted.frame, timestamp, &mut output) { - Ok(()) => { - let encode_duration = encode_start.elapsed(); - let pipeline_latency = converted.submit_time.elapsed(); - metrics.record_frame_encoded(encode_duration, pipeline_latency); - } - Err(_) => {} + if let Ok(()) = + encoder.queue_preconverted_frame(converted.frame, timestamp, &mut output) + { + let encode_duration = encode_start.elapsed(); + let pipeline_latency = converted.submit_time.elapsed(); + metrics.record_frame_encoded(encode_duration, pipeline_latency); } } else { break; @@ -374,8 +370,8 @@ async fn main() { println!("\n=== Frame Rate Test ==="); let (frames, fps, inter_frame_times) = run_camera_frame_rate_test(&camera.0, 3).await; - println!("Frames captured: {}", frames); - println!("Average FPS: {:.1}", fps); + println!("Frames captured: {frames}"); + println!("Average FPS: {fps:.1}"); if !inter_frame_times.is_empty() { let avg_interval: Duration = @@ -384,9 +380,9 @@ async fn main() { let min_interval = inter_frame_times.iter().min().unwrap(); println!("Inter-frame timing:"); - println!(" Average: {:?}", avg_interval); - println!(" Min: {:?}", min_interval); - println!(" Max: {:?}", max_interval); + println!(" Average: {avg_interval:?}"); + println!(" Min: {min_interval:?}"); + println!(" Max: {max_interval:?}"); let mut sorted = inter_frame_times.clone(); sorted.sort(); diff --git a/crates/recording/examples/encoding-benchmark.rs b/crates/recording/examples/encoding-benchmark.rs index 8169e30ff9..d9e62ad59a 100644 --- a/crates/recording/examples/encoding-benchmark.rs +++ b/crates/recording/examples/encoding-benchmark.rs @@ -192,7 +192,7 @@ fn benchmark_conversion_formats(config: &BenchmarkConfig) { println!("\n=== Format Conversion Benchmarks ===\n"); for (input, output, name) in formats { - println!("Testing: {}", name); + println!("Testing: {name}"); let mut cfg = config.clone(); cfg.duration_secs = 5; @@ -406,7 +406,7 @@ fn main() { "encode" => benchmark_encode_times(&config), "workers" => benchmark_worker_counts(&config), "resolutions" => benchmark_resolutions(&config), - "full" | _ => { + _ => { benchmark_conversion_formats(&config); benchmark_encode_times(&config); benchmark_worker_counts(&config); diff --git a/crates/recording/examples/recording-benchmark.rs b/crates/recording/examples/recording-benchmark.rs index 87adbe7da9..13eab946f4 100644 --- a/crates/recording/examples/recording-benchmark.rs +++ b/crates/recording/examples/recording-benchmark.rs @@ -265,7 +265,7 @@ async fn main() -> Result<(), Box> { .unwrap_or(5); stress_test_recording(cycles, config.duration_secs).await?; } - "full" | _ => { + _ => { println!("Mode: Full benchmark suite\n"); println!("--- Screen Recording ---"); diff --git a/crates/recording/tests/hardware_compat.rs b/crates/recording/tests/hardware_compat.rs index d5c940caa9..6900012e6f 100644 --- a/crates/recording/tests/hardware_compat.rs +++ b/crates/recording/tests/hardware_compat.rs @@ -187,7 +187,7 @@ fn test_gpu_detection() { println!(" -> Microsoft WARP: Software rendering/encoding"); } cap_frame_converter::GpuVendor::Unknown(id) => { - println!(" -> Unknown GPU vendor (0x{:04X}): Software fallback", id); + println!(" -> Unknown GPU vendor (0x{id:04X}): Software fallback"); } } } else { @@ -200,7 +200,7 @@ fn test_graphics_capture_support() { test_utils::init_tracing(); let supported = scap_direct3d::is_supported().unwrap_or(false); - println!("Windows Graphics Capture API supported: {}", supported); + println!("Windows Graphics Capture API supported: {supported}"); if !supported { let version = scap_direct3d::WindowsVersion::detect(); @@ -232,7 +232,7 @@ fn test_camera_enumeration() { println!("Device ID: {}", camera.device_id()); if let Some(model_id) = camera.model_id() { - println!("Model ID: {}", model_id); + println!("Model ID: {model_id}"); } if let Some(formats) = camera.formats() { @@ -251,7 +251,7 @@ fn test_camera_enumeration() { for (resolution, frame_rates) in format_summary.iter() { let rates: Vec = frame_rates .iter() - .map(|(_, _, fps)| format!("{:.1}fps", fps)) + .map(|(_, _, fps)| format!("{fps:.1}fps")) .collect(); println!(" {}: {}", resolution, rates.join(", ")); } @@ -289,14 +289,14 @@ fn test_encoder_availability_matrix() { for (name, description) in h264_encoders { let available = ffmpeg::encoder::find_by_name(name).is_some(); let status = if available { "✓" } else { "✗" }; - println!(" {} {} ({})", status, description, name); + println!(" {status} {description} ({name})"); } println!("\n=== HEVC/H.265 Encoder Availability ==="); for (name, description) in hevc_encoders { let available = ffmpeg::encoder::find_by_name(name).is_some(); let status = if available { "✓" } else { "✗" }; - println!(" {} {} ({})", status, description, name); + println!(" {status} {description} ({name})"); } let gpu = cap_frame_converter::detect_primary_gpu(); @@ -374,11 +374,11 @@ fn test_d3d11_converter_capability() { }; println!(" ✓ {}: {} ({:?})", name, hw, result.backend); if let Some(reason) = result.fallback_reason { - println!(" Fallback: {}", reason); + println!(" Fallback: {reason}"); } } Err(e) => { - println!(" ✗ {}: Failed - {}", name, e); + println!(" ✗ {name}: Failed - {e}"); } } } @@ -402,7 +402,7 @@ fn test_supported_pixel_formats() { for (format, name) in formats { let supported = cap_frame_converter::is_format_supported(format); let status = if supported { "✓" } else { "✗" }; - println!(" {} {}", status, name); + println!(" {status} {name}"); } } @@ -560,14 +560,14 @@ fn test_camera_capture_basic() { std::thread::sleep(Duration::from_secs(2)); let frames = frame_count.load(std::sync::atomic::Ordering::Relaxed); - println!("Captured {} frames in 2 seconds", frames); + println!("Captured {frames} frames in 2 seconds"); let _ = handle.stop_capturing(); assert!(frames > 0, "Should have captured at least one frame"); } Err(e) => { - println!("Failed to start capture: {:?}", e); + println!("Failed to start capture: {e:?}"); } } } @@ -678,7 +678,7 @@ fn test_hardware_compatibility_summary() { } else { "? Unknown".to_string() }; - println!("║ Windows: {:<52} ║", windows_status); + println!("║ Windows: {windows_status:<52} ║"); let gpu_status = if let Some(g) = gpu { format!( @@ -689,21 +689,21 @@ fn test_hardware_compatibility_summary() { } else { "⚠ No GPU (WARP software rendering)".to_string() }; - println!("║ GPU: {:<56} ║", gpu_status); + println!("║ GPU: {gpu_status:<56} ║"); let capture_status = if diagnostics.graphics_capture_supported { "✓ Available" } else { "✗ Unavailable" }; - println!("║ Screen Capture: {:<45} ║", capture_status); + println!("║ Screen Capture: {capture_status:<45} ║"); let d3d11_status = if diagnostics.d3d11_video_processor_available { "✓ GPU accelerated" } else { "⚠ CPU fallback (swscale)" }; - println!("║ Frame Conversion: {:<43} ║", d3d11_status); + println!("║ Frame Conversion: {d3d11_status:<43} ║"); let hw_encoders: Vec<&str> = diagnostics .available_encoders @@ -716,11 +716,11 @@ fn test_hardware_compatibility_summary() { } else { "⚠ Software only (libx264)".to_string() }; - println!("║ Encoding: {:<51} ║", encoder_status); + println!("║ Encoding: {encoder_status:<51} ║"); let cameras: Vec = cap_camera::list_cameras().collect(); let camera_status = format!("{} camera(s) detected", cameras.len()); - println!("║ Cameras: {:<52} ║", camera_status); + println!("║ Cameras: {camera_status:<52} ║"); println!("╠════════════════════════════════════════════════════════════════╣"); @@ -792,10 +792,7 @@ fn test_frame_conversion_performance() { let avg_ms = elapsed.as_secs_f64() * 1000.0 / test_iterations as f64; let fps_capacity = 1000.0 / avg_ms; - println!( - "Performance: {:.2}ms/frame avg ({:.1} fps capacity)", - avg_ms, fps_capacity - ); + println!("Performance: {avg_ms:.2}ms/frame avg ({fps_capacity:.1} fps capacity)"); let target_fps = 60.0; let target_ms = 1000.0 / target_fps; @@ -902,7 +899,7 @@ fn test_minimum_requirements_check() { .collect(); if !hw_encoders.is_empty() { - println!(" ✓ Hardware video encoder ({:?})", hw_encoders); + println!(" ✓ Hardware video encoder ({hw_encoders:?})"); } else { println!(" ⚠ No hardware encoders (will use CPU encoding)"); warnings.push("CPU encoding may impact system performance"); @@ -921,7 +918,7 @@ fn test_minimum_requirements_check() { } else if requirements_met { println!("⚠ Requirements met with warnings:"); for warning in &warnings { - println!(" - {}", warning); + println!(" - {warning}"); } } else { println!("✗ Missing required components - Cap may not function correctly"); diff --git a/crates/rendering/src/cpu_yuv.rs b/crates/rendering/src/cpu_yuv.rs index 7dbb0a26bb..ea929e5358 100644 --- a/crates/rendering/src/cpu_yuv.rs +++ b/crates/rendering/src/cpu_yuv.rs @@ -990,11 +990,7 @@ mod tests { let diff = (*s as i32 - *d as i32).abs(); assert!( diff <= 2, - "Mismatch at index {}: scalar={}, simd={}, diff={}", - i, - s, - d, - diff + "Mismatch at index {i}: scalar={s}, simd={d}, diff={diff}" ); } } @@ -1064,11 +1060,7 @@ mod tests { let diff = (*a as i32 - *b as i32).abs(); assert!( diff <= 2, - "Mismatch at index {}: expected={}, got={}, diff={}", - i, - a, - b, - diff + "Mismatch at index {i}: expected={a}, got={b}, diff={diff}" ); } } @@ -1119,11 +1111,7 @@ mod tests { let diff = (*s as i32 - *d as i32).abs(); assert!( diff <= 2, - "YUV420P mismatch at index {}: scalar={}, simd={}, diff={}", - i, - s, - d, - diff + "YUV420P mismatch at index {i}: scalar={s}, simd={d}, diff={diff}" ); } } diff --git a/crates/scap-direct3d/examples/cli.rs b/crates/scap-direct3d/examples/cli.rs index cc8020267b..faaa65a547 100644 --- a/crates/scap-direct3d/examples/cli.rs +++ b/crates/scap-direct3d/examples/cli.rs @@ -5,15 +5,12 @@ fn main() { #[cfg(windows)] mod windows { - use scap_direct3d::{Capturer, PixelFormat, Settings}; - use scap_ffmpeg::*; use scap_targets::*; use std::time::Duration; - use windows::Win32::Graphics::Direct3D11::D3D11_BOX; pub fn main() { let display = Display::primary(); - let display = display.raw_handle(); + let _display = display.raw_handle(); // let mut capturer = Capturer::new( // display.try_as_capture_item().unwrap(), From 4b13bb3592eaed440b0bbb4616af2b52dcd956d9 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 23:29:55 +0000 Subject: [PATCH 19/26] coderabbit --- crates/recording/src/output_pipeline/win.rs | 22 +++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 459ea13e78..78d173ded0 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -275,7 +275,7 @@ impl Muxer for WindowsMuxer { }; let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + timestamp.saturating_sub(first) } else { first_timestamp = Some(timestamp); Duration::ZERO @@ -672,7 +672,7 @@ impl Muxer for WindowsCameraMuxer { )>, > { let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + timestamp.saturating_sub(first) } else { first_timestamp = Some(timestamp); Duration::ZERO @@ -754,7 +754,7 @@ impl Muxer for WindowsCameraMuxer { timestamp: Duration| -> anyhow::Result> { let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + timestamp.saturating_sub(first) } else { first_timestamp = Some(timestamp); Duration::ZERO @@ -936,12 +936,12 @@ pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result>; + let converted_data_storage; let (final_data, final_format): (&[u8], ffmpeg::format::Pixel) = if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - converted_data = Some(convert_uyvy_to_yuyv(data, frame.width, frame.height)); + converted_data_storage = convert_uyvy_to_yuyv(data, frame.width, frame.height); ( - converted_data.as_ref().unwrap(), + converted_data_storage.as_slice(), ffmpeg::format::Pixel::YUYV422, ) } else { @@ -996,14 +996,10 @@ pub fn upload_mf_buffer_to_texture( let lock = buffer_guard.lock()?; let original_data = &*lock; - let converted_buffer: Option>; + let converted_buffer_storage; let data: &[u8] = if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - converted_buffer = Some(convert_uyvy_to_yuyv( - original_data, - frame.width, - frame.height, - )); - converted_buffer.as_ref().unwrap() + converted_buffer_storage = convert_uyvy_to_yuyv(original_data, frame.width, frame.height); + converted_buffer_storage.as_slice() } else { original_data }; From 5dd6696ce9530692cda3812e21316a45550cb51a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 18 Dec 2025 23:50:32 +0000 Subject: [PATCH 20/26] coderabbit --- apps/desktop/src-tauri/src/windows.rs | 51 ++++++++++++++----- crates/enc-mediafoundation/src/video/hevc.rs | 5 +- crates/frame-converter/src/d3d11.rs | 8 ++- crates/recording/examples/camera-benchmark.rs | 16 +++--- crates/recording/src/output_pipeline/win.rs | 13 ++--- .../src/output_pipeline/win_segmented.rs | 17 +++++-- .../output_pipeline/win_segmented_camera.rs | 6 +++ crates/recording/tests/hardware_compat.rs | 6 ++- crates/rendering/src/decoder/ffmpeg.rs | 6 ++- crates/scap-ffmpeg/src/direct3d.rs | 18 ++++++- 10 files changed, 111 insertions(+), 35 deletions(-) diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index a223e0f721..552eae2a64 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -288,8 +288,12 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(600.0, 600.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(600.0, 600.0)) { + warn!("Failed to set Setup window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Setup window on Windows: {}", e); + } } window @@ -458,8 +462,12 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(600.0, 465.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(600.0, 465.0)) { + warn!("Failed to set Settings window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Settings window on Windows: {}", e); + } } window @@ -483,8 +491,12 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(1275.0, 800.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(1275.0, 800.0)) { + warn!("Failed to set Editor window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Editor window on Windows: {}", e); + } } window @@ -508,8 +520,15 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(1240.0, 800.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(1240.0, 800.0)) { + warn!( + "Failed to set ScreenshotEditor window size on Windows: {}", + e + ); + } + if let Err(e) = window.center() { + warn!("Failed to center ScreenshotEditor window on Windows: {}", e); + } } window @@ -534,8 +553,12 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(950.0, 850.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(950.0, 850.0)) { + warn!("Failed to set Upgrade window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Upgrade window on Windows: {}", e); + } } window @@ -560,8 +583,12 @@ impl ShowCapWindow { #[cfg(windows)] { use tauri::LogicalSize; - let _ = window.set_size(LogicalSize::new(580.0, 340.0)); - let _ = window.center(); + if let Err(e) = window.set_size(LogicalSize::new(580.0, 340.0)) { + warn!("Failed to set ModeSelect window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center ModeSelect window on Windows: {}", e); + } } window diff --git a/crates/enc-mediafoundation/src/video/hevc.rs b/crates/enc-mediafoundation/src/video/hevc.rs index 2b4a72ec08..948f80abb6 100644 --- a/crates/enc-mediafoundation/src/video/hevc.rs +++ b/crates/enc-mediafoundation/src/video/hevc.rs @@ -427,7 +427,10 @@ impl HevcEncoder { } } _ => { - panic!("Unknown media event type: {}", event_type.0); + eprintln!( + "[cap-enc-mediafoundation] Ignoring unknown media event type: {}", + event_type.0 + ); } } } diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 36e69371e8..670d04cce5 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -1147,12 +1147,16 @@ impl Drop for D3D11Resources { unsafe { if let Some(handle) = self.input_shared_handle.take() { if !handle.is_invalid() { - let _ = CloseHandle(handle); + if let Err(e) = CloseHandle(handle) { + tracing::error!("Failed to close input shared handle: {:?}", e); + } } } if let Some(handle) = self.output_shared_handle.take() { if !handle.is_invalid() { - let _ = CloseHandle(handle); + if let Err(e) = CloseHandle(handle) { + tracing::error!("Failed to close output shared handle: {:?}", e); + } } } } diff --git a/crates/recording/examples/camera-benchmark.rs b/crates/recording/examples/camera-benchmark.rs index 5ad07ad524..8d99d5a4e9 100644 --- a/crates/recording/examples/camera-benchmark.rs +++ b/crates/recording/examples/camera-benchmark.rs @@ -292,12 +292,16 @@ async fn run_camera_encoding_benchmark( let encode_start = Instant::now(); let timestamp = Duration::from_micros(converted.frame.pts().unwrap_or(0) as u64); - if let Ok(()) = - encoder.queue_preconverted_frame(converted.frame, timestamp, &mut output) - { - let encode_duration = encode_start.elapsed(); - let pipeline_latency = converted.submit_time.elapsed(); - metrics.record_frame_encoded(encode_duration, pipeline_latency); + match encoder.queue_preconverted_frame(converted.frame, timestamp, &mut output) { + Ok(()) => { + let encode_duration = encode_start.elapsed(); + let pipeline_latency = converted.submit_time.elapsed(); + metrics.record_frame_encoded(encode_duration, pipeline_latency); + } + Err(e) => { + warn!("Encode error during drain: {}", e); + metrics.record_dropped_output(); + } } } else { break; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 78d173ded0..0bf6a04357 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -287,9 +287,9 @@ impl Muxer for WindowsMuxer { |output_sample| { let mut output = output.lock().unwrap(); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); + if let Err(e) = muxer.write_sample(&output_sample, &mut output) { + tracing::error!("WriteSample failed: {e}"); + } Ok(()) }, @@ -707,9 +707,10 @@ impl Muxer for WindowsCameraMuxer { }, |output_sample| { let mut output = output.lock().unwrap(); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); + if let Err(e) = muxer.write_sample(&output_sample, &mut output) + { + tracing::error!("Camera WriteSample failed: {e}"); + } Ok(()) }, ); diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs index c83e64b735..5c98f790a4 100644 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ b/crates/recording/src/output_pipeline/win_segmented.rs @@ -520,11 +520,20 @@ impl WindowsSegmentedMuxer { Ok(Some((frame.texture().clone(), frame_time))) }, |output_sample| { - let mut output = output_clone.lock().unwrap(); + let mut output = match output_clone.lock() { + Ok(guard) => guard, + Err(e) => { + error!("Failed to lock output mutex: {e}"); + return Err(windows::core::Error::new( + windows::core::HRESULT(0x80004005u32 as i32), + format!("Mutex poisoned: {e}"), + )); + } + }; - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); + if let Err(e) = muxer.write_sample(&output_sample, &mut output) { + warn!("WriteSample failed: {e}"); + } Ok(()) }, diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 36ec47c86f..5fcae07472 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -587,6 +587,12 @@ impl WindowsSegmentedCameraMuxer { } } + if let Ok(mut output_guard) = output_clone.lock() { + if let Err(e) = encoder.flush(&mut output_guard) { + warn!("Failed to flush software encoder: {e}"); + } + } + Ok(()) } } diff --git a/crates/recording/tests/hardware_compat.rs b/crates/recording/tests/hardware_compat.rs index 6900012e6f..53d7bdb72b 100644 --- a/crates/recording/tests/hardware_compat.rs +++ b/crates/recording/tests/hardware_compat.rs @@ -738,10 +738,12 @@ fn test_hardware_compatibility_summary() { } fn truncate_string(s: &str, max_len: usize) -> String { - if s.len() <= max_len { + if s.chars().count() <= max_len { s.to_string() } else { - format!("{}...", &s[..max_len - 3]) + let truncate_at = max_len.saturating_sub(3); + let truncated: String = s.chars().take(truncate_at).collect(); + format!("{truncated}...") } } diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 8504f14b88..961e4cec28 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -149,8 +149,12 @@ impl FfmpegDecoder { std::thread::spawn(move || { let hw_device_type = if cfg!(target_os = "macos") { Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX) - } else { + } else if cfg!(target_os = "linux") { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI) + } else if cfg!(target_os = "windows") { Some(AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2) + } else { + None }; let mut this = match cap_video_decode::FFmpegDecoder::new(path.clone(), hw_device_type) diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 047e5d4e35..fcfdd34fa6 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -12,8 +12,24 @@ fn copy_frame_data( row_length: usize, height: usize, ) { + debug_assert!(height > 0, "height must be positive"); + debug_assert!( + src_bytes.len() + >= (height - 1) + .saturating_mul(src_stride) + .saturating_add(row_length), + "source buffer too small" + ); + debug_assert!( + dest_bytes.len() + >= (height - 1) + .saturating_mul(dest_stride) + .saturating_add(row_length), + "destination buffer too small" + ); + if src_stride == row_length && dest_stride == row_length { - let total_bytes = row_length * height; + let total_bytes = row_length.saturating_mul(height); unsafe { std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), dest_bytes.as_mut_ptr(), total_bytes); } From 0599ec3d582074a3f93bf71c6df31b33ae928409 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:09:58 +0000 Subject: [PATCH 21/26] coderabbit --- .claude/agents/coderabbit-pr-reviewer.md | 117 ++++++++++++++++++ .claude/settings.local.json | 4 +- .../desktop/src-tauri/src/general_settings.rs | 9 +- crates/enc-ffmpeg/src/video/hevc.rs | 2 +- crates/enc-mediafoundation/src/video/hevc.rs | 2 +- crates/frame-converter/src/d3d11.rs | 2 +- 6 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 .claude/agents/coderabbit-pr-reviewer.md diff --git a/.claude/agents/coderabbit-pr-reviewer.md b/.claude/agents/coderabbit-pr-reviewer.md new file mode 100644 index 0000000000..1bf03b3c3a --- /dev/null +++ b/.claude/agents/coderabbit-pr-reviewer.md @@ -0,0 +1,117 @@ +--- +name: coderabbit-pr-reviewer +description: Use this agent when you need to automatically implement CodeRabbit PR review suggestions from a GitHub pull request. This agent fetches review comments from the GitHub API, parses CodeRabbit's AI agent instructions, and systematically applies the suggested fixes while respecting project conventions.\n\nExamples:\n\n\nContext: User wants to implement CodeRabbit suggestions from a specific PR\nuser: "Implement the CodeRabbit suggestions from PR #1459"\nassistant: "I'll use the coderabbit-pr-reviewer agent to fetch and implement the CodeRabbit suggestions from PR #1459"\n\nSince the user wants to implement CodeRabbit suggestions, use the coderabbit-pr-reviewer agent to handle the complete workflow of fetching, parsing, and implementing the suggestions.\n\n\n\n\nContext: User mentions CodeRabbit review comments need to be addressed\nuser: "There are some CodeRabbit review comments on the PR that need fixing"\nassistant: "I'll launch the coderabbit-pr-reviewer agent to systematically implement the CodeRabbit review suggestions"\n\nThe user is referencing CodeRabbit review comments that need implementation. Use the coderabbit-pr-reviewer agent to handle this workflow.\n\n\n\n\nContext: User wants to address automated code review feedback\nuser: "Can you fix the issues that CodeRabbit found in CapSoftware/Cap pull request 1500?"\nassistant: "I'll use the coderabbit-pr-reviewer agent to fetch the CodeRabbit comments from PR #1500 in CapSoftware/Cap and implement the suggested fixes"\n\nThe user explicitly mentions CodeRabbit and a specific PR. Use the coderabbit-pr-reviewer agent to process these suggestions.\n\n +model: opus +color: red +--- + +You are an expert code review implementation agent specializing in automatically applying CodeRabbit PR review suggestions. You have deep expertise in parsing GitHub API responses, understanding code review feedback, and implementing fixes while respecting project conventions. + +## Your Mission + +You systematically fetch, parse, and implement CodeRabbit review suggestions from GitHub pull requests, adapting each fix to work within the project's existing architecture and dependencies. + +## Workflow + +### Phase 1: Fetch CodeRabbit Comments + +1. Determine the repository owner, repo name, and PR number from user input +2. Fetch PR review comments using the GitHub API: + - Endpoint: `GET https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/comments` + - Filter for comments where `user.login == "coderabbitai[bot]"` +3. Extract key fields from each comment: + - `path`: The file to modify + - `line` or `original_line`: The line number + - `body`: The full markdown comment with instructions + +### Phase 2: Parse Each Comment + +For each CodeRabbit comment: + +1. **Extract the AI Agent Instructions** + - Look for the section: `
🤖 Prompt for AI Agents` + - Parse the specific instructions within this block + +2. **Extract the Suggested Fix** + - Look for the section: `
🔧 Suggested fix` + - Parse the diff blocks showing old vs new code + +3. **Understand the Issue Context** + - Note the issue type (⚠️ Potential issue, 📌 Major, etc.) + - Read the description explaining why the change is needed + +### Phase 3: Implement Each Fix + +For each suggestion: + +1. **Read Context** + - Open the target file at the specified line + - Read surrounding context (±10 lines) + - Check the project's `Cargo.toml` or `package.json` for available dependencies + +2. **Adapt the Fix** + - Apply the suggested diff + - If suggested imports/crates don't exist, use alternatives: + - `tracing::warn!` → `eprintln!` (if tracing unavailable) + - `tracing::error!` → `eprintln!` (if tracing unavailable) + - `anyhow::Error` → `Box` (if anyhow unavailable) + - Respect project conventions (especially the NO COMMENTS rule for this codebase) + +3. **Common Fix Patterns** + - Silent Result handling: Replace `let _ = result` with `if let Err(e) = result { warn!(...) }` + - Panic prevention: Replace `panic!()` with warning logs and graceful handling + - Missing flush calls: Add explicit flush before returns + - UTF-8 safety: Use `.chars().take()` instead of byte slicing + - Platform handling: Add cfg-based platform branches + +### Phase 4: Validate Changes + +After implementing all fixes: + +1. **Format Code** + - Rust: `cargo fmt --all` + - TypeScript: `pnpm format` + +2. **Check Compilation** + - Rust: `cargo check -p affected_crate` + - TypeScript: `pnpm typecheck` + +3. **Lint Check** + - Rust: `cargo clippy` + - TypeScript: `pnpm lint` + +## Critical Rules + +1. **Never add code comments** - This project forbids all forms of comments. Code must be self-explanatory through naming, types, and structure. + +2. **Verify dependencies exist** before using them. Check Cargo.toml/package.json first. + +3. **Preserve existing code style** - Match the patterns used in surrounding code. + +4. **Skip conflicting suggestions** - If a CodeRabbit suggestion conflicts with project rules (like adding comments), skip it and report to the user. + +5. **Report unresolvable issues** - Some suggestions may require manual review. Document these clearly. + +## Output Format + +After completing implementation, provide: + +1. **Summary of Changes** + - List each file modified + - Brief description of each fix applied + +2. **Skipped Suggestions** + - Any suggestions that couldn't be implemented automatically + - Reason for skipping + +3. **Validation Results** + - Formatting status + - Compilation status + - Any remaining warnings or errors + +## Error Handling + +- If GitHub API fails: Report the error and suggest checking authentication or rate limits +- If a file doesn't exist: Skip that suggestion and note it in the report +- If compilation fails after a fix: Attempt to diagnose, or revert and report for manual review +- If no CodeRabbit comments found: Inform the user and suggest verifying the PR number diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 8b8848cd81..1db421f36f 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -46,7 +46,9 @@ "Bash(cargo clean:*)", "Bash(cargo test:*)", "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")", - "Bash(cargo build:*)" + "Bash(cargo build:*)", + "Bash(gh api:*)", + "Bash(curl:*)" ], "deny": [], "ask": [] diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index f5169a7a27..1b8c89c77f 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -72,12 +72,11 @@ pub struct GeneralSettingsStore { pub hide_dock_icon: bool, #[serde(default)] pub auto_create_shareable_link: bool, - #[serde(default = "true_b")] + #[serde(default = "default_true")] pub enable_notifications: bool, #[serde(default)] pub disable_auto_open_links: bool, - // first launch: store won't exist so show startup - #[serde(default = "true_b")] + #[serde(default = "default_true")] pub has_completed_startup: bool, #[serde(default)] pub theme: AppTheme, @@ -206,10 +205,6 @@ pub enum AppTheme { Dark, } -fn true_b() -> bool { - true -} - impl GeneralSettingsStore { pub fn get(app: &AppHandle) -> Result, String> { match app.store("store").map(|s| s.get("general_settings")) { diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rs index f9548959bc..0f9e24ca96 100644 --- a/crates/enc-ffmpeg/src/video/hevc.rs +++ b/crates/enc-ffmpeg/src/video/hevc.rs @@ -243,7 +243,7 @@ impl HevcEncoderBuilder { let bitrate = get_bitrate( output_width, output_height, - input_config.frame_rate.0 as f32 / input_config.frame_rate.1 as f32, + input_config.frame_rate.0 as f32 / input_config.frame_rate.1.max(1) as f32, bpp, ); diff --git a/crates/enc-mediafoundation/src/video/hevc.rs b/crates/enc-mediafoundation/src/video/hevc.rs index 948f80abb6..e24e45330f 100644 --- a/crates/enc-mediafoundation/src/video/hevc.rs +++ b/crates/enc-mediafoundation/src/video/hevc.rs @@ -420,7 +420,7 @@ impl HevcEncoder { consecutive_empty_samples += 1; if consecutive_empty_samples > MAX_CONSECUTIVE_EMPTY_SAMPLES { return Err(windows::core::Error::new( - windows::core::HRESULT(0), + windows::core::HRESULT(-1), "Too many consecutive empty samples", )); } diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 670d04cce5..06e957bb51 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -815,7 +815,7 @@ impl FrameConverter for D3D11Converter { .fetch_add(elapsed_ns, Ordering::Relaxed); let frame_count = count + 1; - if frame_count > 0 && frame_count % 300 == 0 { + if frame_count % 300 == 0 { let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; tracing::debug!( From 1993e7383aa25d40ef6177477a1b40e05d9c0591 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:14:16 +0000 Subject: [PATCH 22/26] clippy --- apps/desktop/src-tauri/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 2b1d08cff5..e914955174 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1232,7 +1232,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> { Command::new("explorer") .args(["/select,", path_str]) .spawn() - .map_err(|e| format!("Failed to open folder: {}", e))?; + .map_err(|e| format!("Failed to open folder: {e}"))?; } #[cfg(target_os = "macos")] @@ -1254,7 +1254,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> { .ok_or("Invalid path")?, ) .spawn() - .map_err(|e| format!("Failed to open folder: {}", e))?; + .map_err(|e| format!("Failed to open folder: {e}"))?; } Ok(()) From 344956b849d4e0de4df848667c576fc5c4e992ab Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:28:23 +0000 Subject: [PATCH 23/26] coderabbit --- .claude/settings.local.json | 4 +++- Cargo.lock | 1 + apps/desktop/src-tauri/src/general_settings.rs | 2 +- crates/enc-mediafoundation/Cargo.toml | 1 + crates/enc-mediafoundation/src/video/hevc.rs | 5 +---- crates/recording/src/output_pipeline/win.rs | 5 ++++- .../recording/src/output_pipeline/win_segmented_camera.rs | 7 ++++++- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 1db421f36f..b83de1a96a 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -48,7 +48,9 @@ "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")", "Bash(cargo build:*)", "Bash(gh api:*)", - "Bash(curl:*)" + "Bash(curl:*)", + "Bash(node -e:*)", + "Bash(findstr:*)" ], "deny": [], "ask": [] diff --git a/Cargo.lock b/Cargo.lock index 1cbb176ce2..4aa78c2b5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1360,6 +1360,7 @@ dependencies = [ "scap-direct3d", "scap-targets", "thiserror 1.0.69", + "tracing", "windows 0.60.0", "windows-core 0.60.1", "windows-numerics 0.2.0", diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 1b8c89c77f..e970af964d 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -123,7 +123,7 @@ pub struct GeneralSettingsStore { pub instant_mode_max_resolution: u32, #[serde(default)] pub default_project_name_template: Option, - #[serde(default = "default_true")] + #[serde(default)] pub crash_recovery_recording: bool, } diff --git a/crates/enc-mediafoundation/Cargo.toml b/crates/enc-mediafoundation/Cargo.toml index 7df3e9a3cb..cac0676628 100644 --- a/crates/enc-mediafoundation/Cargo.toml +++ b/crates/enc-mediafoundation/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" cap-media-info = { path = "../media-info" } futures.workspace = true thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/enc-mediafoundation/src/video/hevc.rs b/crates/enc-mediafoundation/src/video/hevc.rs index e24e45330f..c99e476fa7 100644 --- a/crates/enc-mediafoundation/src/video/hevc.rs +++ b/crates/enc-mediafoundation/src/video/hevc.rs @@ -427,10 +427,7 @@ impl HevcEncoder { } } _ => { - eprintln!( - "[cap-enc-mediafoundation] Ignoring unknown media event type: {}", - event_type.0 - ); + tracing::warn!("Ignoring unknown media event type: {}", event_type.0); } } } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 0bf6a04357..18d7107906 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -285,7 +285,10 @@ impl Muxer for WindowsMuxer { Ok(Some((frame.texture().clone(), frame_time))) }, |output_sample| { - let mut output = output.lock().unwrap(); + let Ok(mut output) = output.lock() else { + tracing::error!("Failed to lock output mutex - poisoned"); + return Ok(()); + }; if let Err(e) = muxer.write_sample(&output_sample, &mut output) { tracing::error!("WriteSample failed: {e}"); diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 5fcae07472..7f57cd92c8 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -516,7 +516,12 @@ impl WindowsSegmentedCameraMuxer { Ok(Some((texture, duration_to_timespan(relative)))) }, |output_sample| { - let mut output = output_clone.lock().unwrap(); + let mut output = output_clone.lock().map_err(|e| { + windows::core::Error::new( + windows::core::HRESULT(-1), + format!("Mutex poisoned: {e}"), + ) + })?; muxer .write_sample(&output_sample, &mut output) .map_err(|e| { From ddfe183fc6700ad896a3530928883d50eba61883 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:34:16 +0000 Subject: [PATCH 24/26] clippy --- crates/frame-converter/src/d3d11.rs | 24 +++++++++++------------- crates/rendering/src/decoder/mod.rs | 4 +++- crates/scap-direct3d/src/lib.rs | 9 +++++---- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 06e957bb51..fccc462ab7 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -815,7 +815,7 @@ impl FrameConverter for D3D11Converter { .fetch_add(elapsed_ns, Ordering::Relaxed); let frame_count = count + 1; - if frame_count % 300 == 0 { + if frame_count.is_multiple_of(300) { let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; tracing::debug!( @@ -1145,19 +1145,17 @@ unsafe impl Sync for D3D11Converter {} impl Drop for D3D11Resources { fn drop(&mut self) { unsafe { - if let Some(handle) = self.input_shared_handle.take() { - if !handle.is_invalid() { - if let Err(e) = CloseHandle(handle) { - tracing::error!("Failed to close input shared handle: {:?}", e); - } - } + if let Some(handle) = self.input_shared_handle.take() + && !handle.is_invalid() + && let Err(e) = CloseHandle(handle) + { + tracing::error!("Failed to close input shared handle: {:?}", e); } - if let Some(handle) = self.output_shared_handle.take() { - if !handle.is_invalid() { - if let Err(e) = CloseHandle(handle) { - tracing::error!("Failed to close output shared handle: {:?}", e); - } - } + if let Some(handle) = self.output_shared_handle.take() + && !handle.is_invalid() + && let Err(e) = CloseHandle(handle) + { + tracing::error!("Failed to close output shared handle: {:?}", e); } } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 4ba74a146c..bad0ddbaf1 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -6,7 +6,9 @@ use std::{ time::Duration, }; use tokio::sync::oneshot; -use tracing::{debug, info, warn}; +#[cfg(target_os = "windows")] +use tracing::warn; +use tracing::{debug, info}; #[cfg(target_os = "macos")] mod avassetreader; diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index 1a2dffaab5..c631f149af 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -105,10 +105,11 @@ impl StagingTexturePool { let index = self.next_index.fetch_add(1, Ordering::Relaxed) % STAGING_POOL_SIZE; - if let Some(pooled) = textures.get(index) { - if pooled.width == width && pooled.height == height { - return Ok(pooled.texture.clone()); - } + if let Some(pooled) = textures.get(index) + && pooled.width == width + && pooled.height == height + { + return Ok(pooled.texture.clone()); } let texture_desc = D3D11_TEXTURE2D_DESC { From 1aac5a658997af96debfb534f26c9d198deaa8de Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:39:57 +0000 Subject: [PATCH 25/26] clippy --- crates/enc-mediafoundation/src/video/h264.rs | 44 ++++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 129ba0024f..9611b8632d 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -542,16 +542,16 @@ impl H264Encoder { let mut should_exit = false; while !should_exit { let health_status = health_monitor.check_health(); - if !health_status.is_healthy { - if let Some(reason) = health_status.failure_reason { - let _ = self.cleanup_encoder(); - return Err(EncoderRuntimeError::EncoderUnhealthy { - reason, - inputs_without_output: health_status.inputs_without_output, - process_failures: health_status.consecutive_process_failures, - frames_encoded: health_status.total_frames_encoded, - }); - } + if !health_status.is_healthy + && let Some(reason) = health_status.failure_reason + { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status.inputs_without_output, + process_failures: health_status.consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); } let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; @@ -588,18 +588,18 @@ impl H264Encoder { Err(_) => { health_monitor.record_process_failure(); let health_status = health_monitor.check_health(); - if !health_status.is_healthy { - if let Some(reason) = health_status.failure_reason { - let _ = self.cleanup_encoder(); - return Err(EncoderRuntimeError::EncoderUnhealthy { - reason, - inputs_without_output: health_status - .inputs_without_output, - process_failures: health_status - .consecutive_process_failures, - frames_encoded: health_status.total_frames_encoded, - }); - } + if !health_status.is_healthy + && let Some(reason) = health_status.failure_reason + { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status + .inputs_without_output, + process_failures: health_status + .consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); } should_exit = false; } From 30cf231f56cedcff7ede8ad07900ab113f8b1071 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:46:56 +0000 Subject: [PATCH 26/26] clippy --- .../recording/src/output_pipeline/win_segmented_camera.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 7f57cd92c8..71fbf6fcd6 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -592,10 +592,10 @@ impl WindowsSegmentedCameraMuxer { } } - if let Ok(mut output_guard) = output_clone.lock() { - if let Err(e) = encoder.flush(&mut output_guard) { - warn!("Failed to flush software encoder: {e}"); - } + if let Ok(mut output_guard) = output_clone.lock() + && let Err(e) = encoder.flush(&mut output_guard) + { + warn!("Failed to flush software encoder: {e}"); } Ok(())