Skip to content

Commit 3d65d06

Browse files
richiemcilroyrichiemcilroy
authored
Merge pull request #1707 from CapSoftware/codex/editor-long-video-performance
2 parents 477951f + cf27e33 commit 3d65d06

16 files changed

Lines changed: 425 additions & 170 deletions

‎apps/desktop/src-tauri/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ tauri = { workspace = true, features = [
2424
"protocol-asset",
2525
"tray-icon",
2626
"image-png",
27-
"devtools",
2827
] }
2928
tauri-specta = { version = "=2.0.0-rc.20", features = ["derive", "typescript"] }
3029
tauri-plugin-dialog = "2.2.0"

‎apps/desktop/src-tauri/src/lib.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2764,11 +2764,15 @@ async fn get_display_frame_for_cropping(
27642764
use cap_rendering::{PixelFormat, cpu_yuv};
27652765
use image::{ImageEncoder, codecs::png::PngEncoder};
27662766
use std::io::Cursor;
2767+
use std::time::Instant;
2768+
2769+
let total_started_at = Instant::now();
27672770

27682771
let frame_number = editor_instance.state.lock().await.playhead_position;
27692772
let time_secs = frame_number as f64 / fps as f64;
27702773

27712774
let project = editor_instance.project_config.1.borrow().clone();
2775+
let lookup_started_at = Instant::now();
27722776

27732777
let (segment_time, segment) = project
27742778
.get_segment_time(time_secs)
@@ -2785,19 +2789,23 @@ async fn get_display_frame_for_cropping(
27852789
.find(|v| v.index == segment.recording_clip)
27862790
.map(|v| v.offsets)
27872791
.unwrap_or(ClipOffsets::default());
2792+
let lookup_elapsed_ms = lookup_started_at.elapsed().as_secs_f64() * 1000.0;
27882793

2794+
let decode_started_at = Instant::now();
27892795
let segment_frames = segment_medias
27902796
.decoders
27912797
.get_frames(segment_time as f32, false, true, clip_offsets)
27922798
.await
27932799
.ok_or_else(|| "Failed to get frame".to_string())?;
2800+
let decode_elapsed_ms = decode_started_at.elapsed().as_secs_f64() * 1000.0;
27942801

27952802
let screen_frame = segment_frames
27962803
.screen_frame
27972804
.ok_or_else(|| "Failed to get screen frame".to_string())?;
27982805
let width = screen_frame.width();
27992806
let height = screen_frame.height();
28002807

2808+
let convert_started_at = Instant::now();
28012809
let rgba_data = match screen_frame.format() {
28022810
PixelFormat::Rgba => screen_frame.data().to_vec(),
28032811
PixelFormat::Nv12 => {
@@ -2833,12 +2841,31 @@ async fn get_display_frame_for_cropping(
28332841
rgba
28342842
}
28352843
};
2844+
let convert_elapsed_ms = convert_started_at.elapsed().as_secs_f64() * 1000.0;
28362845

2846+
let encode_started_at = Instant::now();
28372847
let mut png_data = Cursor::new(Vec::new());
28382848
let encoder = PngEncoder::new(&mut png_data);
28392849
encoder
28402850
.write_image(&rgba_data, width, height, image::ExtendedColorType::Rgba8)
28412851
.map_err(|e| format!("Failed to encode PNG: {e}"))?;
2852+
let encode_elapsed_ms = encode_started_at.elapsed().as_secs_f64() * 1000.0;
2853+
let total_elapsed_ms = total_started_at.elapsed().as_secs_f64() * 1000.0;
2854+
2855+
debug!(
2856+
target: "cap_crop_profile",
2857+
frame_number = frame_number,
2858+
time_secs = time_secs,
2859+
segment_time = segment_time,
2860+
width = width,
2861+
height = height,
2862+
lookup_ms = lookup_elapsed_ms,
2863+
decode_ms = decode_elapsed_ms,
2864+
convert_ms = convert_elapsed_ms,
2865+
encode_ms = encode_elapsed_ms,
2866+
total_ms = total_elapsed_ms,
2867+
"crop frame profile"
2868+
);
28422869

28432870
Ok(png_data.into_inner())
28442871
}

‎apps/desktop/src-tauri/src/windows.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2120,7 +2120,8 @@ impl ShowCapWindow {
21202120
.visible(false)
21212121
.accept_first_mouse(true)
21222122
.shadow(true)
2123-
.theme(theme);
2123+
.theme(theme)
2124+
.devtools(cfg!(debug_assertions));
21242125

21252126
if !id.is_transparent() {
21262127
let is_dark = match theme {

‎apps/desktop/src/routes/editor/Editor.tsx‎

Lines changed: 204 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
Match,
1919
on,
2020
onCleanup,
21+
onMount,
2122
Show,
2223
Switch,
2324
} from "solid-js";
@@ -59,9 +60,16 @@ const MIN_PLAYER_CONTENT_HEIGHT = 320;
5960
const MIN_TIMELINE_HEIGHT = 240;
6061
const RESIZE_HANDLE_HEIGHT = 16;
6162
const MIN_PLAYER_HEIGHT = MIN_PLAYER_CONTENT_HEIGHT + RESIZE_HANDLE_HEIGHT;
62-
6363
const TIMELINE_RESIZE_GRIP_MARKS = [0, 1, 2] as const;
6464

65+
function logCropProfile(
66+
stage: string,
67+
data: Record<string, number | string | boolean | null> = {},
68+
) {
69+
if (!import.meta.env.DEV) return;
70+
console.info("[crop-profile]", stage, data);
71+
}
72+
6573
function getEditorErrorMessage(error: unknown) {
6674
return error instanceof Error ? error.message : String(error);
6775
}
@@ -772,24 +780,182 @@ function Dialogs() {
772780
const [crop, setCrop] = createSignal(CROP_ZERO);
773781
const [aspect, setAspect] = createSignal<Ratio | null>(null);
774782

783+
const initialPreviewUrl = dialog().previewUrl ?? null;
775784
const [frameBlobUrl, setFrameBlobUrl] = createSignal<
776785
string | null
777-
>(null);
786+
>(initialPreviewUrl);
787+
const [frameSource, setFrameSource] = createSignal<
788+
"captured-preview" | "accurate-frame" | "screenshot"
789+
>(initialPreviewUrl ? "captured-preview" : "screenshot");
790+
const cropOpenedAt = performance.now();
791+
const screenshotSrc = convertFileSrc(
792+
`${editorInstance.path}/screenshots/display.jpg`,
793+
);
794+
795+
let cancelled = false;
796+
let frameLoadDelayTimeoutId:
797+
| ReturnType<typeof globalThis.setTimeout>
798+
| undefined;
799+
let frameLoadTimeoutId:
800+
| ReturnType<typeof globalThis.setTimeout>
801+
| undefined;
802+
let frameLoadIdleId: number | undefined;
803+
let accurateFrameRequested = false;
804+
const idleWindow = globalThis as typeof globalThis & {
805+
requestIdleCallback?: (
806+
callback: () => void,
807+
options?: { timeout?: number },
808+
) => number;
809+
cancelIdleCallback?: (handle: number) => void;
810+
};
811+
812+
const clearScheduledAccurateFrame = () => {
813+
if (frameLoadDelayTimeoutId !== undefined) {
814+
globalThis.clearTimeout(frameLoadDelayTimeoutId);
815+
frameLoadDelayTimeoutId = undefined;
816+
}
817+
if (frameLoadIdleId !== undefined) {
818+
idleWindow.cancelIdleCallback?.(frameLoadIdleId);
819+
frameLoadIdleId = undefined;
820+
}
821+
if (frameLoadTimeoutId !== undefined) {
822+
globalThis.clearTimeout(frameLoadTimeoutId);
823+
frameLoadTimeoutId = undefined;
824+
}
825+
};
826+
827+
const setPreviewBlob = (
828+
blob: Blob,
829+
source: "accurate-frame",
830+
) => {
831+
const nextUrl = URL.createObjectURL(blob);
832+
const previousUrl = frameBlobUrl();
833+
setFrameBlobUrl(nextUrl);
834+
setFrameSource(source);
835+
if (previousUrl) {
836+
URL.revokeObjectURL(previousUrl);
837+
}
838+
};
839+
840+
const requestAccurateFrame = (reason: string) => {
841+
if (accurateFrameRequested || cancelled) return;
778842

779-
commands
780-
.getDisplayFrameForCropping(FPS)
781-
.then((pngBytes) => {
782-
const blob = new Blob([new Uint8Array(pngBytes)], {
783-
type: "image/png",
843+
clearScheduledAccurateFrame();
844+
845+
accurateFrameRequested = true;
846+
const frameRequestStartedAt = performance.now();
847+
logCropProfile("accurate-frame-request-start", {
848+
elapsedMs: Number(
849+
(frameRequestStartedAt - cropOpenedAt).toFixed(2),
850+
),
851+
reason,
852+
});
853+
854+
void commands
855+
.getDisplayFrameForCropping(FPS)
856+
.then((pngBytes) => {
857+
if (cancelled) return;
858+
859+
setPreviewBlob(
860+
new Blob([new Uint8Array(pngBytes)], {
861+
type: "image/png",
862+
}),
863+
"accurate-frame",
864+
);
865+
logCropProfile("accurate-frame-request-finish", {
866+
elapsedMs: Number(
867+
(performance.now() - cropOpenedAt).toFixed(2),
868+
),
869+
requestMs: Number(
870+
(performance.now() - frameRequestStartedAt).toFixed(
871+
2,
872+
),
873+
),
874+
reason,
875+
});
876+
})
877+
.catch((error: unknown) => {
878+
if (cancelled) return;
879+
console.warn("Display frame fetch failed:", error);
880+
logCropProfile("accurate-frame-request-failed", {
881+
elapsedMs: Number(
882+
(performance.now() - cropOpenedAt).toFixed(2),
883+
),
884+
requestMs: Number(
885+
(performance.now() - frameRequestStartedAt).toFixed(
886+
2,
887+
),
888+
),
889+
message:
890+
error instanceof Error
891+
? error.message
892+
: String(error),
893+
reason,
894+
});
784895
});
785-
const url = URL.createObjectURL(blob);
786-
setFrameBlobUrl(url);
787-
})
788-
.catch((error: unknown) => {
789-
console.warn("Display frame fetch failed:", error);
896+
};
897+
898+
const scheduleAccurateFrame = (
899+
reason: string,
900+
options: {
901+
delayMs?: number;
902+
idleTimeoutMs: number;
903+
fallbackDelayMs: number;
904+
},
905+
) => {
906+
const queueIdleFrame = () => {
907+
const loadFrame = () => requestAccurateFrame(reason);
908+
909+
if (idleWindow.requestIdleCallback) {
910+
frameLoadIdleId = idleWindow.requestIdleCallback(
911+
() => {
912+
frameLoadIdleId = undefined;
913+
loadFrame();
914+
},
915+
{
916+
timeout: options.idleTimeoutMs,
917+
},
918+
);
919+
return;
920+
}
921+
922+
frameLoadTimeoutId = globalThis.setTimeout(() => {
923+
frameLoadTimeoutId = undefined;
924+
loadFrame();
925+
}, options.fallbackDelayMs);
926+
};
927+
928+
if (!options.delayMs) {
929+
queueIdleFrame();
930+
return;
931+
}
932+
933+
frameLoadDelayTimeoutId = globalThis.setTimeout(() => {
934+
frameLoadDelayTimeoutId = undefined;
935+
if (cancelled || accurateFrameRequested) return;
936+
queueIdleFrame();
937+
}, options.delayMs);
938+
};
939+
940+
onMount(() => {
941+
logCropProfile("dialog-mounted", {
942+
elapsedMs: Number(
943+
(performance.now() - cropOpenedAt).toFixed(2),
944+
),
945+
recordingDurationSec: Math.round(
946+
editorInstance.recordingDuration,
947+
),
948+
});
949+
950+
scheduleAccurateFrame("immediate", {
951+
idleTimeoutMs: 500,
952+
fallbackDelayMs: 16,
790953
});
954+
});
791955

792956
onCleanup(() => {
957+
cancelled = true;
958+
clearScheduledAccurateFrame();
793959
const url = frameBlobUrl();
794960
if (url) {
795961
URL.revokeObjectURL(url);
@@ -959,12 +1125,33 @@ function Dialogs() {
9591125
<img
9601126
class="shadow pointer-events-none max-h-[70vh]"
9611127
alt="Current frame"
962-
src={
963-
frameBlobUrl() ??
964-
convertFileSrc(
965-
`${editorInstance.path}/screenshots/display.jpg`,
966-
)
1128+
onError={() => {
1129+
const failedSource = frameSource();
1130+
logCropProfile("preview-image-failed", {
1131+
elapsedMs: Number(
1132+
(performance.now() - cropOpenedAt).toFixed(
1133+
2,
1134+
),
1135+
),
1136+
source: failedSource,
1137+
});
1138+
requestAccurateFrame(
1139+
failedSource === "screenshot"
1140+
? "screenshot-load-failed"
1141+
: "preview-load-failed",
1142+
);
1143+
}}
1144+
onLoad={() =>
1145+
logCropProfile("preview-image-loaded", {
1146+
elapsedMs: Number(
1147+
(performance.now() - cropOpenedAt).toFixed(
1148+
2,
1149+
),
1150+
),
1151+
source: frameSource(),
1152+
})
9671153
}
1154+
src={frameBlobUrl() ?? screenshotSrc}
9681155
/>
9691156
</Cropper>
9701157
</div>

0 commit comments

Comments
 (0)