feat: Improve audio latency handling and buffer sizing

crates/editor/src/playback.rs

@@ -245,13 +245,26 @@ impl AudioPlayback {
         let mut elapsed = 0;
         let mut prev_audio_config = project.borrow().audio.clone();
 
+        let mut latency_correction_secs: Option<f64> = None;
+        let sample_rate_f64 = output_info.sample_rate as f64;
+
         let stream_result = device.build_output_stream(
             &config,
-            move |buffer: &mut [T], _info| {
+            move |buffer: &mut [T], info| {
                 let project = project.borrow();
 
+                if latency_correction_secs.is_none() {
+                    latency_correction_secs = info
+                        .timestamp()
+                        .playback
+                        .duration_since(&info.timestamp().callback)
+                        .map(|latency| latency.as_secs_f64());
+                }
+
+                let latency_secs = latency_correction_secs.unwrap_or_default();
+
                 audio_renderer.set_playhead(
-                    playhead + elapsed as f64 / output_info.sample_rate as f64,
+                    playhead + latency_secs + elapsed as f64 / sample_rate_f64,
                     &project,
                 );
                 prev_audio_config = project.audio.clone();

crates/media-info/src/lib.rs

@@ -55,12 +55,19 @@ impl AudioInfo {
     }
 
     pub fn from_stream_config(config: &SupportedStreamConfig) -> Self {
+        Self::from_stream_config_with_buffer(config, None)
+    }
+
+    pub fn from_stream_config_with_buffer(
+        config: &SupportedStreamConfig,
+        buffer_size_override: Option<u32>,
+    ) -> Self {
         let sample_format = ffmpeg_sample_format_for(config.sample_format()).unwrap();
-        let buffer_size = match config.buffer_size() {
+        let buffer_size = buffer_size_override.unwrap_or_else(|| match config.buffer_size() {
             SupportedBufferSize::Range { max, .. } => *max,
             // TODO: Different buffer sizes for different contexts?
             SupportedBufferSize::Unknown => 1024,
-        };
+        });
 
         let raw_channels = config.channels();
         let channels = if Self::channel_layout_raw(raw_channels).is_some() {

crates/recording/src/feeds/microphone.rs

@@ -1,7 +1,8 @@
 use cap_media_info::{AudioInfo, ffmpeg_sample_format_for};
 use cap_timestamp::Timestamp;
 use cpal::{
-    Device, InputCallbackInfo, SampleFormat, StreamError, SupportedStreamConfig,
+    BufferSize, Device, InputCallbackInfo, SampleFormat, StreamError, SupportedStreamConfig,
+    SupportedStreamConfigRange,
     traits::{DeviceTrait, HostTrait, StreamTrait},
 };
 use flume::TrySendError;
@@ -61,6 +62,7 @@ impl OpenState {
             self.attached = Some(AttachedState {
                 id: data.id,
                 config: data.config.clone(),
+                buffer_size_frames: data.buffer_size_frames,
                 done_tx: data.done_tx,
             });
             self.connecting = None;
@@ -77,6 +79,7 @@ struct AttachedState {
     #[allow(dead_code)]
     id: u32,
     config: SupportedStreamConfig,
+    buffer_size_frames: Option<u32>,
     done_tx: mpsc::SyncSender<()>,
 }
 
@@ -146,23 +149,79 @@ fn get_usable_device(device: Device) -> Option<(String, Device, SupportedStreamC
                     .then(b.max_sample_rate().cmp(&a.max_sample_rate()))
             });
 
-            configs
-                .into_iter()
-                .filter(|c| c.min_sample_rate().0 <= 48000 && c.max_sample_rate().0 >= 48000)
-                .find(|c| ffmpeg_sample_format_for(c.sample_format()).is_some())
+            configs.into_iter().find_map(|config| {
+                ffmpeg_sample_format_for(config.sample_format())
+                    .map(|_| config.with_sample_rate(select_sample_rate(&config)))
+            })
         });
 
-    result.and_then(|config| {
-        let final_config = config.with_sample_rate(cpal::SampleRate(48000));
-        device.name().ok().map(|name| (name, device, final_config))
-    })
+    result.and_then(|config| device.name().ok().map(|name| (name, device, config)))
+}
+
+fn select_sample_rate(config: &SupportedStreamConfigRange) -> cpal::SampleRate {
+    const PREFERRED_RATES: [u32; 2] = [48_000, 44_100];
+
+    for rate in PREFERRED_RATES {
+        if config.min_sample_rate().0 <= rate && config.max_sample_rate().0 >= rate {
+            return cpal::SampleRate(rate);
+        }
+    }
+
+    cpal::SampleRate(config.max_sample_rate().0)
+}
+
+const TARGET_LATENCY_MS: u32 = 35;
+const MIN_LATENCY_MS: u32 = 10;
+const MAX_LATENCY_MS: u32 = 120;
+const ABS_MIN_BUFFER_FRAMES: u32 = 128;
+
+fn stream_config_with_latency(config: &SupportedStreamConfig) -> (cpal::StreamConfig, Option<u32>) {
+    let mut stream_config: cpal::StreamConfig = config.clone().into();
+    let buffer_size_frames = desired_buffer_size_frames(config);
+
+    if let Some(frames) = buffer_size_frames {
+        stream_config.buffer_size = BufferSize::Fixed(frames);
+    }
+
+    (stream_config, buffer_size_frames)
+}
+
+fn desired_buffer_size_frames(config: &SupportedStreamConfig) -> Option<u32> {
+    match config.buffer_size() {
+        cpal::SupportedBufferSize::Range { min, max } => {
+            let sample_rate = config.sample_rate().0;
+
+            if sample_rate == 0 || *max == 0 {
+                return None;
+            }
+
+            let mut desired = latency_ms_to_frames(sample_rate, TARGET_LATENCY_MS);
+            let min_latency_frames = latency_ms_to_frames(sample_rate, MIN_LATENCY_MS);
+            let max_latency_frames = latency_ms_to_frames(sample_rate, MAX_LATENCY_MS);
+
+            desired = desired.max(min_latency_frames);
+            desired = desired.max(ABS_MIN_BUFFER_FRAMES.min(*max));
+            desired = desired.max(*min);
+            desired = desired.min(*max);
+            desired = desired.min(max_latency_frames.max(*min));
+
+            Some(desired)
+        }
+        cpal::SupportedBufferSize::Unknown => None,
+    }
+}
+
+fn latency_ms_to_frames(sample_rate: u32, milliseconds: u32) -> u32 {
+    let frames = (sample_rate as u64 * milliseconds as u64) / 1_000;
+    frames.max(1) as u32
 }
 
 #[derive(Reply)]
 pub struct MicrophoneFeedLock {
     actor: ActorRef<MicrophoneFeed>,
     config: SupportedStreamConfig,
     audio_info: AudioInfo,
+    buffer_size_frames: Option<u32>,
     drop_tx: Option<oneshot::Sender<()>>,
 }
 
@@ -174,6 +233,10 @@ impl MicrophoneFeedLock {
     pub fn audio_info(&self) -> AudioInfo {
         self.audio_info
     }
+
+    pub fn buffer_size_frames(&self) -> Option<u32> {
+        self.buffer_size_frames
+    }
 }
 
 impl Deref for MicrophoneFeedLock {
@@ -209,6 +272,7 @@ pub struct Lock;
 struct InputConnected {
     id: u32,
     config: SupportedStreamConfig,
+    buffer_size_frames: Option<u32>,
     done_tx: SyncSender<()>,
 }
 
@@ -256,17 +320,20 @@ impl Message<SetInput> for MicrophoneFeed {
         };
 
         let sample_format = config.sample_format();
+        let (stream_config, buffer_size_frames) = stream_config_with_latency(&config);
 
-        let (ready_tx, ready_rx) = oneshot::channel();
+        let (ready_tx, ready_rx) = oneshot::channel::<Result<Option<u32>, SetInputError>>();
         let (done_tx, done_rx) = mpsc::sync_channel(0);
 
         let actor_ref = ctx.actor_ref();
         let ready = {
-            let config = config.clone();
+            let config_for_ready = config.clone();
             ready_rx
-                .map(|v| {
+                .map(move |v| {
+                    let config = config_for_ready.clone();
                     v.map_err(|_| SetInputError::BuildStreamCrashed)
-                        .map(|_| config)
+                        .and_then(|inner| inner)
+                        .map(|buffer_size| (config, buffer_size))
                 })
                 .shared()
         };
@@ -279,9 +346,10 @@ impl Message<SetInput> for MicrophoneFeed {
                 ready
                     .clone()
                     .map(move |v| {
-                        v.map(|config| InputConnected {
+                        v.map(|(config, buffer_size_frames)| InputConnected {
                             id,
                             config,
+                            buffer_size_frames,
                             done_tx,
                         })
                     })
@@ -291,6 +359,7 @@ impl Message<SetInput> for MicrophoneFeed {
 
         std::thread::spawn({
             let config = config.clone();
+            let stream_config = stream_config.clone();
             let device_name_for_log = device.name().ok();
             move || {
                 // Log all configs for debugging
@@ -307,16 +376,26 @@ impl Message<SetInput> for MicrophoneFeed {
                     }
                 }
 
+                let buffer_size_description = match &stream_config.buffer_size {
+                    BufferSize::Default => "default".to_string(),
+                    BufferSize::Fixed(frames) => format!(
+                        "{} frames (~{:.1}ms)",
+                        frames,
+                        (*frames as f64 / config.sample_rate().0 as f64) * 1000.0
+                    ),
+                };
+
                 info!(
-                    "🎤 Building stream for '{:?}' with config: rate={}, channels={}, format={:?}",
+                    "🎤 Building stream for '{:?}' with config: rate={}, channels={}, format={:?}, buffer_size={}",
                     device_name_for_log,
                     config.sample_rate().0,
                     config.channels(),
-                    sample_format
+                    sample_format,
+                    buffer_size_description
                 );
 
                 let stream = match device.build_input_stream_raw(
-                    &config.into(),
+                    &stream_config,
                     sample_format,
                     {
                         let actor_ref = actor_ref.clone();
@@ -361,7 +440,7 @@ impl Message<SetInput> for MicrophoneFeed {
                     return;
                 }
 
-                let _ = ready_tx.send(Ok(()));
+                let _ = ready_tx.send(Ok(buffer_size_frames));
 
                 match done_rx.recv() {
                     Ok(_) => {
@@ -377,13 +456,15 @@ impl Message<SetInput> for MicrophoneFeed {
         tokio::spawn({
             let ready = ready.clone();
             let actor = ctx.actor_ref();
+            let done_tx = done_tx;
             async move {
                 match ready.await {
-                    Ok(config) => {
+                    Ok((config, buffer_size_frames)) => {
                         let _ = actor
                             .tell(InputConnected {
                                 id,
                                 config,
+                                buffer_size_frames,
                                 done_tx,
                             })
                             .await;
@@ -395,7 +476,9 @@ impl Message<SetInput> for MicrophoneFeed {
             }
         });
 
-        Ok(ready.boxed())
+        let ready_for_return = ready.clone().map(|result| result.map(|(config, _)| config));
+
+        Ok(ready_for_return.boxed())
     }
 }
 
@@ -481,6 +564,7 @@ impl Message<Lock> for MicrophoneFeed {
         };
 
         let config = attached.config.clone();
+        let buffer_size_frames = attached.buffer_size_frames;
 
         self.state = State::Locked { inner: attached };
 
@@ -493,9 +577,10 @@ impl Message<Lock> for MicrophoneFeed {
         });
 
         Ok(MicrophoneFeedLock {
-            audio_info: AudioInfo::from_stream_config(&config),
+            audio_info: AudioInfo::from_stream_config_with_buffer(&config, buffer_size_frames),
             actor: ctx.actor_ref(),
             config,
+            buffer_size_frames,
             drop_tx: Some(drop_tx),
         })
     }