diff --git a/README.md b/README.md index 7718c40686..bb155f05fb 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,8 @@ agent-browser state clean --older-than # Delete old states With recording `--cursor`, the pointer and click ripple render with the page, keeping drags synchronized in every captured frame. The temporary overlay is inert, hidden from accessibility snapshots, and removed when recording stops. Screenshots taken during the recording include it. +Contact sheets sample candidate frames at the rate set by `--fps`. At rates below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered. + ### Navigation ```bash diff --git a/cli/src/mcp.rs b/cli/src/mcp.rs index 88e7931c6b..2d56973594 100644 --- a/cli/src/mcp.rs +++ b/cli/src/mcp.rs @@ -1396,7 +1396,7 @@ fn parity_tools() -> Vec { "description": "Capture rate in frames per second (default 30, max 60).", }, "cursor": { "type": "boolean", "description": "Render a pointer and click ripple with the page so drags stay synchronized. The inert overlay is hidden from accessibility snapshots, included in screenshots while recording, and removed on stop." }, - "contactSheet": { "type": "boolean", "description": "Export first, changed, and final frames as a timestamped PNG beside the video." }, + "contactSheet": { "type": "boolean", "description": "Export first, changed, and final frames as a timestamped PNG beside the video. Candidate frames are sampled at fps. Below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered." }, "contactSheetThreshold": { "type": "number", "minimum": 0, "maximum": 1, "description": "Changed-pixel ratio required to select a contact-sheet frame (default 0.05). Implies contactSheet." }, }), &["path"], @@ -1425,7 +1425,7 @@ fn parity_tools() -> Vec { "description": "Capture rate in frames per second (default 30, max 60).", }, "cursor": { "type": "boolean", "description": "Render a pointer and click ripple with the page so drags stay synchronized. The inert overlay is hidden from accessibility snapshots, included in screenshots while recording, and removed on stop." }, - "contactSheet": { "type": "boolean", "description": "Export first, changed, and final frames as a timestamped PNG beside the video." }, + "contactSheet": { "type": "boolean", "description": "Export first, changed, and final frames as a timestamped PNG beside the video. Candidate frames are sampled at fps. Below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered." }, "contactSheetThreshold": { "type": "number", "minimum": 0, "maximum": 1, "description": "Changed-pixel ratio required to select a contact-sheet frame (default 0.05). Implies contactSheet." }, }), &["path"], @@ -4913,6 +4913,19 @@ mod tests { tool["inputSchema"]["properties"]["contactSheet"]["type"], "boolean" ); + let contact_sheet_description = tool["inputSchema"]["properties"]["contactSheet"] + ["description"] + .as_str() + .unwrap(); + for needle in ["sampled at fps", "final captured frame"] { + assert!( + contact_sheet_description.contains(needle), + "{} contact-sheet description should mention {}: {}", + name, + needle, + contact_sheet_description + ); + } let threshold = &tool["inputSchema"]["properties"]["contactSheetThreshold"]; assert_eq!(threshold["minimum"], json!(0)); assert_eq!(threshold["maximum"], json!(1)); diff --git a/cli/src/native/recording.rs b/cli/src/native/recording.rs index 0ff4b38966..1154e34638 100644 --- a/cli/src/native/recording.rs +++ b/cli/src/native/recording.rs @@ -1399,7 +1399,7 @@ struct CapturedVideoFrame { async fn seed_recording_outputs( frame_tx: &mpsc::Sender, - contact_tx: Option<&std::sync::mpsc::SyncSender>, + contact_tx: Option<&std::sync::mpsc::SyncSender<(CapturedVideoFrame, tokio::time::Instant)>>, frame: CapturedVideoFrame, ) -> Result<(), String> { frame_tx @@ -1407,15 +1407,7 @@ async fn seed_recording_outputs( .await .map_err(|_| "Recording encoder stopped unexpectedly".to_string())?; if let Some(contact_tx) = contact_tx { - contact_tx.try_send(frame).map_err(|error| match error { - std::sync::mpsc::TrySendError::Full(_) => format!( - "Contact sheet analyzer fell behind by more than {} buffered frames", - ENCODER_FRAME_BUFFER - ), - std::sync::mpsc::TrySendError::Disconnected(_) => { - "Contact sheet analyzer stopped unexpectedly".to_string() - } - })?; + send_contact_frame(contact_tx, frame)?; } Ok(()) } @@ -1514,12 +1506,13 @@ pub fn spawn_recording_task( let captured = match started { Ok(_) => { + let contact_sink = contact_tx.map(|tx| ContactFrameSink::new(tx, fps)); collect_frames( &client, &capture_session, events, frame_tx, - contact_tx, + contact_sink, &shared_captured, cancel_rx, ) @@ -1633,7 +1626,7 @@ async fn collect_frames( capture_session: &str, mut events: mpsc::Receiver, frame_tx: mpsc::Sender, - contact_tx: Option>, + mut contact_sink: Option, shared_captured: &AtomicU64, cancel_rx: oneshot::Receiver<()>, ) -> Result<(), String> { @@ -1683,16 +1676,8 @@ async fn collect_frames( "Recording encoder stopped unexpectedly".to_string() } })?; - if let Some(contact_tx) = contact_tx.as_ref() { - contact_tx.try_send(frame).map_err(|error| match error { - std::sync::mpsc::TrySendError::Full(_) => format!( - "Contact sheet analyzer fell behind by more than {} buffered frames", - ENCODER_FRAME_BUFFER - ), - std::sync::mpsc::TrySendError::Disconnected(_) => { - "Contact sheet analyzer stopped unexpectedly".to_string() - } - })?; + if let Some(contact_sink) = contact_sink.as_mut() { + contact_sink.consider(frame)?; } } } else if event.method == "Inspector.detached" { @@ -1702,12 +1687,91 @@ async fn collect_frames( } } } + if let Some(contact_sink) = contact_sink { + contact_sink.finish()?; + } Ok(()) } +struct ContactFrameSink { + tx: std::sync::mpsc::SyncSender<(CapturedVideoFrame, tokio::time::Instant)>, + governor: ContactFrameGovernor, + pending: Option, +} + +impl ContactFrameSink { + fn new( + tx: std::sync::mpsc::SyncSender<(CapturedVideoFrame, tokio::time::Instant)>, + fps: u32, + ) -> Self { + Self { + tx, + governor: ContactFrameGovernor::new(fps), + pending: None, + } + } + + fn consider(&mut self, frame: CapturedVideoFrame) -> Result<(), String> { + if self.governor.should_send(frame.elapsed) { + send_contact_frame(&self.tx, frame)?; + self.pending = None; + } else { + self.pending = Some(frame); + } + Ok(()) + } + + fn finish(self) -> Result<(), String> { + if let Some(frame) = self.pending { + send_contact_frame(&self.tx, frame)?; + } + Ok(()) + } +} + +struct ContactFrameGovernor { + period: Duration, + next_allowed: Duration, +} + +impl ContactFrameGovernor { + fn new(fps: u32) -> Self { + let period = frame_period(fps); + Self { + period, + next_allowed: period, + } + } + + fn should_send(&mut self, elapsed: Duration) -> bool { + if elapsed < self.next_allowed { + return false; + } + self.next_allowed = elapsed.saturating_add(self.period); + true + } +} + +fn send_contact_frame( + contact_tx: &std::sync::mpsc::SyncSender<(CapturedVideoFrame, tokio::time::Instant)>, + frame: CapturedVideoFrame, +) -> Result<(), String> { + contact_tx + .try_send((frame, tokio::time::Instant::now())) + .map_err(|error| match error { + std::sync::mpsc::TrySendError::Full(_) => format!( + "Contact sheet analyzer fell behind by more than {} buffered frames", + ENCODER_FRAME_BUFFER + ), + std::sync::mpsc::TrySendError::Disconnected(_) => { + "Contact sheet analyzer stopped unexpectedly".to_string() + } + }) +} + /// Analyze frames and render finalized cells on the blocking worker. fn collect_contact_frames( - frames: std::sync::mpsc::Receiver, + frames: std::sync::mpsc::Receiver<(CapturedVideoFrame, tokio::time::Instant)>, threshold: f64, cursor: bool, shared_cursor: &SharedRecordingCursor, @@ -1716,14 +1780,15 @@ fn collect_contact_frames( let mut max_lag = Duration::ZERO; let mut processed = 0u64; loop { - let frame = match frames.recv_timeout(Duration::from_millis(CONTACT_SHEET_BURST_QUIET_MS)) { - Ok(frame) => frame, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - collector.flush_pending(); - continue; - } - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, - }; + let (frame, queued_at) = + match frames.recv_timeout(Duration::from_millis(CONTACT_SHEET_BURST_QUIET_MS)) { + Ok(frame) => frame, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + collector.flush_pending(); + continue; + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, + }; let cursor_state = if cursor { shared_cursor .lock() @@ -1740,7 +1805,7 @@ fn collect_contact_frames( frame.device_height, ); // Measure through completed selection/rendering, not just dequeue. - let lag = frame.captured_at.elapsed(); + let lag = queued_at.elapsed(); max_lag = max_lag.max(lag); processed += 1; if lag > MAX_ENCODER_LAG { @@ -3027,6 +3092,60 @@ mod tests { assert_eq!(frame_period(60), Duration::from_micros(16_666)); } + #[test] + fn contact_sheet_frames_obey_recording_fps_governor() { + let (tx, rx) = std::sync::mpsc::sync_channel(10); + let mut sink = ContactFrameSink::new(tx, 1); + for (sequence, elapsed_ms) in [100, 999, 1_000, 1_100, 2_000, 2_100] + .into_iter() + .enumerate() + { + sink.consider(CapturedVideoFrame { + sequence: sequence as u64, + image_data: Arc::new(Vec::new()), + elapsed: Duration::from_millis(elapsed_ms), + captured_at: tokio::time::Instant::now(), + timestamp: 0.0, + device_width: 0.0, + device_height: 0.0, + }) + .unwrap(); + } + sink.finish().unwrap(); + + assert_eq!( + rx.try_iter() + .map(|(frame, _)| frame.sequence) + .collect::>(), + vec![2, 4, 5] + ); + } + + #[test] + fn quiet_page_final_frame_does_not_count_governor_wait_as_analysis_lag() { + let image = image::RgbImage::from_pixel(64, 64, image::Rgb([12, 34, 56])); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, image::ImageFormat::Png).unwrap(); + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let mut sink = ContactFrameSink::new(tx, 1); + sink.consider(CapturedVideoFrame { + sequence: 1, + image_data: Arc::new(png.into_inner()), + elapsed: Duration::from_millis(100), + captured_at: tokio::time::Instant::now() - MAX_ENCODER_LAG - Duration::from_millis(1), + timestamp: 0.0, + device_width: 64.0, + device_height: 64.0, + }) + .unwrap(); + sink.finish().unwrap(); + + let cursor = Arc::new(Mutex::new(RecordingCursorHistory::default())); + let frames = collect_contact_frames(rx, DEFAULT_CONTACT_SHEET_THRESHOLD, false, &cursor) + .expect("an intentional FPS-governor wait must not count as analysis lag"); + assert_eq!(frames.len(), 1); + } + #[tokio::test] async fn test_spawn_ffmpeg_reports_missing_binary() { let mut command = tokio::process::Command::new("agent-browser-no-such-ffmpeg"); diff --git a/cli/src/output.rs b/cli/src/output.rs index a9bd0da005..1662125c6d 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -2942,6 +2942,10 @@ With --cursor, an inert overlay renders the pointer and page together so drags stay synchronized. It is hidden from accessibility snapshots and removed on stop. Screenshots taken while recording include the overlay. +Contact sheets sample candidates at the requested recording rate. At rates +below 60 fps, brief UI states between samples may not appear; the final +captured frame is always considered. + Operations: start [url] Start recording the active page (navigates first if url given) stop Stop recording and save video diff --git a/docs/src/app/recording/page.mdx b/docs/src/app/recording/page.mdx index bdae0cc110..54c6048685 100644 --- a/docs/src/app/recording/page.mdx +++ b/docs/src/app/recording/page.mdx @@ -60,7 +60,7 @@ agent-browser record start ./checkout.webm --contact-sheet agent-browser record start ./checkout.webm --contact-sheet-threshold 0.02 ``` -The threshold accepts `0` to `1`, defaults to `0.05`, and implies `--contact-sheet`. Contact sheets contain at most 100 frames. +The threshold accepts `0` to `1`, defaults to `0.05`, and implies `--contact-sheet`. Candidate frames are sampled at the rate set by `--fps`. At rates below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered. Contact sheets contain at most 100 frames. ![Example contact sheet with timestamps and highlighted change regions](/recording/contact-sheet-example.png) diff --git a/skill-data/core/SKILL.md b/skill-data/core/SKILL.md index 9bfc44ce1a..81426b7178 100644 --- a/skill-data/core/SKILL.md +++ b/skill-data/core/SKILL.md @@ -386,7 +386,7 @@ agent-browser click @e3 agent-browser record stop ``` -Recording uses the active tab. Use `--cursor` for an animated pointer, `--contact-sheet` for a visual summary, and `--fps 60` for motion-heavy recordings. The cursor renders with the page so drags stay synchronized. Its inert overlay is hidden from accessibility snapshots, included in screenshots while recording, and removed on stop. +Recording uses the active tab. Use `--cursor` for an animated pointer, `--contact-sheet` for a visual summary, and `--fps 60` for motion-heavy recordings. The cursor renders with the page so drags stay synchronized. Its inert overlay is hidden from accessibility snapshots, included in screenshots while recording, and removed on stop. Contact sheets sample candidate frames at the rate set by `--fps`. At rates below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered. See [references/video-recording.md](references/video-recording.md) for frame rate guidance, codec options, and more. diff --git a/skill-data/core/references/video-recording.md b/skill-data/core/references/video-recording.md index 6c73d867c1..e05d805eb4 100644 --- a/skill-data/core/references/video-recording.md +++ b/skill-data/core/references/video-recording.md @@ -109,7 +109,7 @@ agent-browser record start ./checkout.webm --contact-sheet agent-browser record start ./checkout.webm --contact-sheet-threshold 0.02 ``` -The threshold accepts values from `0` to `1` and defaults to `0.05`. Passing `--contact-sheet-threshold` implies `--contact-sheet`. At most 100 frames are included. +The threshold accepts values from `0` to `1` and defaults to `0.05`. Passing `--contact-sheet-threshold` implies `--contact-sheet`. Candidate frames are sampled at the rate set by `--fps`. At rates below 60 fps, brief UI states between samples may not appear. The final captured frame is always considered. At most 100 frames are included. ## Use Cases