Merge branch 'main' into pr/1679

Author: richiemcilroy

Cargo.lock

@@ -169,6 +169,7 @@ impl Export {
             compression: cap_export::mp4::ExportCompression::Maximum,
             custom_bpp: None,
             force_ffmpeg_decoder: false,
+            optimize_filesize: false,
         }
         .export(exporter_base, move |_f| {
             // print!("\rrendered frame {f}");

apps/cli/src/main.rs

@@ -1,6 +1,6 @@
 [package]
 name = "cap-desktop"
-version = "0.4.8"
+version = "0.4.82"
 description = "Beautiful screen recordings, owned by you."
 authors = ["you"]
 edition = "2024"

apps/desktop/src-tauri/Cargo.toml

@@ -22,7 +22,7 @@ use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextPar
 
 pub use cap_project::{CaptionSegment, CaptionSettings, CaptionWord};
 
-use crate::http_client;
+use crate::{general_settings::GeneralSettingsStore, http_client};
 
 #[derive(Debug, Serialize, Deserialize, Type, Clone)]
 pub struct CaptionData {
@@ -529,6 +529,7 @@ fn process_with_whisper(
     audio_path: &PathBuf,
     context: Arc<WhisperContext>,
     language: &str,
+    transcription_hints: &[String],
 ) -> Result<CaptionData, String> {
     log::info!("=== WHISPER TRANSCRIPTION START ===");
     log::info!("Processing audio file: {audio_path:?}");
@@ -544,6 +545,10 @@ fn process_with_whisper(
     params.set_language(Some(if language == "auto" { "auto" } else { language }));
     params.set_max_len(i32::MAX);
 
+    if let Some(initial_prompt) = build_initial_prompt(transcription_hints) {
+        params.set_initial_prompt(&initial_prompt);
+    }
+
     log::info!("Whisper params - translate: false, token_timestamps: true, max_len: MAX");
 
     let mut audio_file = File::open(audio_path)
@@ -783,10 +788,32 @@ fn process_with_whisper(
     })
 }
 
+fn build_initial_prompt(transcription_hints: &[String]) -> Option<String> {
+    let mut normalized = Vec::new();
+
+    for hint in transcription_hints {
+        let value = hint.replace('\0', "").trim().to_string();
+        if value.is_empty() || normalized.contains(&value) {
+            continue;
+        }
+        normalized.push(value);
+    }
+
+    if normalized.is_empty() {
+        None
+    } else {
+        Some(format!(
+            "Preferred spellings, names, and capitalization for this transcript: {}",
+            normalized.join("; ")
+        ))
+    }
+}
+
 #[tauri::command]
 #[specta::specta]
 #[instrument]
 pub async fn transcribe_audio(
+    app: AppHandle,
     video_path: String,
     model_path: String,
     language: String,
@@ -843,11 +870,18 @@ pub async fn transcribe_audio(
         }
     };
 
+    let transcription_hints = GeneralSettingsStore::get(&app)
+        .ok()
+        .flatten()
+        .map(|settings| settings.transcription_hints)
+        .unwrap_or_default();
+
     log::info!("Starting Whisper transcription in blocking task...");
-    let whisper_result =
-        tokio::task::spawn_blocking(move || process_with_whisper(&audio_path, context, &language))
-            .await
-            .map_err(|e| format!("Whisper task panicked: {e}"))?;
+    let whisper_result = tokio::task::spawn_blocking(move || {
+        process_with_whisper(&audio_path, context, &language, &transcription_hints)
+    })
+    .await
+    .map_err(|e| format!("Whisper task panicked: {e}"))?;
 
     match whisper_result {
         Ok(captions) => {

apps/desktop/src-tauri/src/captions.rs

@@ -293,6 +293,7 @@ pub async fn generate_export_preview(
         .iter()
         .map(|s| RenderSegment {
             cursor: s.cursor.clone(),
+            keyboard: s.keyboard.clone(),
             decoders: s.decoders.clone(),
         })
         .collect();

apps/desktop/src-tauri/src/export.rs

@@ -27,45 +27,6 @@ fn pack_frame_data(
     data
 }
 
-fn pack_nv12_frame_ref(
-    data: &[u8],
-    width: u32,
-    height: u32,
-    y_stride: u32,
-    frame_number: u32,
-    target_time_ns: u64,
-) -> Vec<u8> {
-    let metadata_size = 28;
-    let mut output = Vec::with_capacity(data.len() + metadata_size);
-    output.extend_from_slice(data);
-    output.extend_from_slice(&y_stride.to_le_bytes());
-    output.extend_from_slice(&height.to_le_bytes());
-    output.extend_from_slice(&width.to_le_bytes());
-    output.extend_from_slice(&frame_number.to_le_bytes());
-    output.extend_from_slice(&target_time_ns.to_le_bytes());
-    output.extend_from_slice(&NV12_FORMAT_MAGIC.to_le_bytes());
-    output
-}
-
-fn pack_frame_data_ref(
-    data: &[u8],
-    stride: u32,
-    height: u32,
-    width: u32,
-    frame_number: u32,
-    target_time_ns: u64,
-) -> Vec<u8> {
-    let metadata_size = 24;
-    let mut output = Vec::with_capacity(data.len() + metadata_size);
-    output.extend_from_slice(data);
-    output.extend_from_slice(&stride.to_le_bytes());
-    output.extend_from_slice(&height.to_le_bytes());
-    output.extend_from_slice(&width.to_le_bytes());
-    output.extend_from_slice(&frame_number.to_le_bytes());
-    output.extend_from_slice(&target_time_ns.to_le_bytes());
-    output
-}
-
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum WSFrameFormat {
     Rgba,
@@ -85,25 +46,33 @@ pub struct WSFrame {
     pub created_at: Instant,
 }
 
-fn pack_ws_frame_ref(frame: &WSFrame) -> Vec<u8> {
+fn pack_ws_frame(frame: &WSFrame) -> Vec<u8> {
+    let metadata_size = match frame.format {
+        WSFrameFormat::Nv12 => 28usize,
+        WSFrameFormat::Rgba => 24,
+    };
+    let mut buf = Vec::with_capacity(frame.data.len() + metadata_size);
+    buf.extend_from_slice(&frame.data);
+
     match frame.format {
-        WSFrameFormat::Nv12 => pack_nv12_frame_ref(
-            &frame.data,
-            frame.width,
-            frame.height,
-            frame.stride,
-            frame.frame_number,
-            frame.target_time_ns,
-        ),
-        WSFrameFormat::Rgba => pack_frame_data_ref(
-            &frame.data,
-            frame.stride,
-            frame.height,
-            frame.width,
-            frame.frame_number,
-            frame.target_time_ns,
-        ),
+        WSFrameFormat::Nv12 => {
+            buf.extend_from_slice(&frame.stride.to_le_bytes());
+            buf.extend_from_slice(&frame.height.to_le_bytes());
+            buf.extend_from_slice(&frame.width.to_le_bytes());
+            buf.extend_from_slice(&frame.frame_number.to_le_bytes());
+            buf.extend_from_slice(&frame.target_time_ns.to_le_bytes());
+            buf.extend_from_slice(&NV12_FORMAT_MAGIC.to_le_bytes());
+        }
+        WSFrameFormat::Rgba => {
+            buf.extend_from_slice(&frame.stride.to_le_bytes());
+            buf.extend_from_slice(&frame.height.to_le_bytes());
+            buf.extend_from_slice(&frame.width.to_le_bytes());
+            buf.extend_from_slice(&frame.frame_number.to_le_bytes());
+            buf.extend_from_slice(&frame.target_time_ns.to_le_bytes());
+        }
     }
+
+    buf
 }
 
 pub async fn create_watch_frame_ws(
@@ -138,7 +107,7 @@ pub async fn create_watch_frame_ws(
         {
             let packed = {
                 let borrowed = camera_rx.borrow();
-                borrowed.as_deref().map(pack_ws_frame_ref)
+                borrowed.as_deref().map(pack_ws_frame)
             };
             if let Some(packed) = packed
                 && let Err(e) = socket.send(Message::Binary(packed)).await
@@ -173,7 +142,7 @@ pub async fn create_watch_frame_ws(
                             WSFrameFormat::Rgba => "RGBA",
                         };
 
-                        let packed = pack_ws_frame_ref(frame);
+                        let packed = pack_ws_frame(frame);
                         let packed_len = packed.len();
 
                         match socket.send(Message::Binary(packed)).await {

apps/desktop/src-tauri/src/frame_ws.rs

@@ -121,8 +121,10 @@ pub struct GeneralSettingsStore {
         skip_serializing_if = "no"
     )]
     pub enable_native_camera_preview: bool,
-    #[serde(default)]
+    #[serde(default = "default_true")]
     pub auto_zoom_on_clicks: bool,
+    #[serde(default = "default_true")]
+    pub capture_keyboard_events: bool,
     #[serde(default)]
     pub post_deletion_behaviour: PostDeletionBehaviour,
     #[serde(default = "default_excluded_windows")]
@@ -137,6 +139,8 @@ pub struct GeneralSettingsStore {
     pub crash_recovery_recording: bool,
     #[serde(default = "default_max_fps")]
     pub max_fps: u32,
+    #[serde(default = "default_transcription_hints")]
+    pub transcription_hints: Vec<String>,
     #[serde(default)]
     pub editor_preview_quality: EditorPreviewQuality,
     #[serde(default)]
@@ -145,6 +149,8 @@ pub struct GeneralSettingsStore {
     pub camera_window_position: Option<WindowPosition>,
     #[serde(default)]
     pub camera_window_positions_by_monitor_name: BTreeMap<String, WindowPosition>,
+    #[serde(default = "default_true")]
+    pub has_completed_onboarding: bool,
 }
 
 fn default_enable_native_camera_preview() -> bool {
@@ -167,6 +173,15 @@ fn default_max_fps() -> u32 {
     60
 }
 
+fn default_transcription_hints() -> Vec<String> {
+    vec![
+        "Cap".to_string(),
+        "TypeScript".to_string(),
+        "My Brand Name".to_string(),
+        "mywebsite.com".to_string(),
+    ]
+}
+
 fn default_server_url() -> String {
     std::option_env!("VITE_SERVER_URL")
         .unwrap_or("https://cap.so")
@@ -203,17 +218,20 @@ impl Default for GeneralSettingsStore {
             recording_countdown: Some(3),
             enable_native_camera_preview: default_enable_native_camera_preview(),
             auto_zoom_on_clicks: false,
+            capture_keyboard_events: true,
             post_deletion_behaviour: PostDeletionBehaviour::DoNothing,
             excluded_windows: default_excluded_windows(),
             delete_instant_recordings_after_upload: false,
             instant_mode_max_resolution: 1920,
             default_project_name_template: None,
             crash_recovery_recording: true,
             max_fps: 60,
+            transcription_hints: default_transcription_hints(),
             editor_preview_quality: EditorPreviewQuality::Half,
             main_window_position: None,
             camera_window_position: None,
             camera_window_positions_by_monitor_name: BTreeMap::new(),
+            has_completed_onboarding: false,
         }
     }
 }

apps/desktop/src-tauri/src/general_settings.rs

@@ -506,6 +506,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result<
                     mic: None,
                     system_audio: None,
                     cursor: None,
+                    keyboard: None,
                 }],
                 cursors: Cursors::default(),
                 status: Some(StudioRecordingStatus::InProgress),
@@ -599,6 +600,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result<
                                     mic: None,
                                     system_audio,
                                     cursor: None,
+                                    keyboard: None,
                                 }],
                                 cursors: Cursors::default(),
                                 status: Some(StudioRecordingStatus::Complete),