Skip to content

Commit 53a5be4

Browse files
richiemcilroyrichiemcilroy
committed
Add enhanced audio processing and playback support
1 parent 82c35dd commit 53a5be4

File tree

13 files changed

+739
-113
lines changed

13 files changed

+739
-113
lines changed

‎apps/desktop/src/routes/(window-chrome)/new-main/index.tsx‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1427,10 +1427,10 @@ function Page() {
14271427
}
14281428

14291429
if (rawOptions.cameraID && "ModelID" in rawOptions.cameraID)
1430-
setCamera.mutate({ ModelID: rawOptions.cameraID.ModelID });
1430+
setCamera.mutate({ model: { ModelID: rawOptions.cameraID.ModelID } });
14311431
else if (rawOptions.cameraID && "DeviceID" in rawOptions.cameraID)
1432-
setCamera.mutate({ DeviceID: rawOptions.cameraID.DeviceID });
1433-
else setCamera.mutate(null);
1432+
setCamera.mutate({ model: { DeviceID: rawOptions.cameraID.DeviceID } });
1433+
else setCamera.mutate({ model: null });
14341434
});
14351435

14361436
const license = createLicenseQuery();
@@ -1444,9 +1444,10 @@ function Page() {
14441444
options={devices.cameras}
14451445
value={options.camera() ?? null}
14461446
onChange={(c) => {
1447-
if (!c) setCamera.mutate(null);
1448-
else if (c.model_id) setCamera.mutate({ ModelID: c.model_id });
1449-
else setCamera.mutate({ DeviceID: c.device_id });
1447+
if (!c) setCamera.mutate({ model: null });
1448+
else if (c.model_id)
1449+
setCamera.mutate({ model: { ModelID: c.model_id } });
1450+
else setCamera.mutate({ model: { DeviceID: c.device_id } });
14501451
}}
14511452
permissions={devices.permissions}
14521453
onOpen={() => {
@@ -1886,10 +1887,10 @@ function Page() {
18861887
selectedTarget={options.camera() ?? null}
18871888
isLoading={devices.isPending}
18881889
onSelect={(c) => {
1889-
if (!c) setCamera.mutate(null);
1890+
if (!c) setCamera.mutate({ model: null });
18901891
else if (c.model_id)
1891-
setCamera.mutate({ ModelID: c.model_id });
1892-
else setCamera.mutate({ DeviceID: c.device_id });
1892+
setCamera.mutate({ model: { ModelID: c.model_id } });
1893+
else setCamera.mutate({ model: { DeviceID: c.device_id } });
18931894
setCameraMenuOpen(false);
18941895
}}
18951896
disabled={isRecording()}

‎apps/web/app/api/playlist/route.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,18 @@ const getPlaylistResponse = (
121121
);
122122
}
123123

124+
if (
125+
Option.isSome(urlParams.fileType) &&
126+
urlParams.fileType.value === "enhanced-audio"
127+
) {
128+
const enhancedAudioKey = `${video.ownerId}/${video.id}/enhanced-audio.mp3`;
129+
return yield* s3.getSignedObjectUrl(enhancedAudioKey).pipe(
130+
Effect.map(HttpServerResponse.redirect),
131+
Effect.catchTag("S3Error", () => new HttpApiError.NotFound()),
132+
Effect.withSpan("fetchEnhancedAudio"),
133+
);
134+
}
135+
124136
yield* Effect.log("Resolving path with custom bucket");
125137

126138
const videoPrefix = `${video.ownerId}/${video.id}/video/`;

‎apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
MediaPlayerCaptions,
1919
MediaPlayerControls,
2020
MediaPlayerControlsOverlay,
21+
MediaPlayerEnhancedAudio,
2122
MediaPlayerError,
2223
MediaPlayerFullscreen,
2324
MediaPlayerLoading,
@@ -48,6 +49,8 @@ function getProgressStatusText(
4849
}
4950
}
5051

52+
type EnhancedAudioStatus = "PROCESSING" | "COMPLETE" | "ERROR" | "SKIPPED";
53+
5154
interface Props {
5255
videoSrc: string;
5356
videoId: Video.VideoId;
@@ -69,6 +72,8 @@ interface Props {
6972
authorName?: string | null;
7073
}>;
7174
onSeek?: (time: number) => void;
75+
enhancedAudioUrl?: string | null;
76+
enhancedAudioStatus?: EnhancedAudioStatus | null;
7277
}
7378

7479
export function CapVideoPlayer({
@@ -86,6 +91,8 @@ export function CapVideoPlayer({
8691
disableCommentStamps = false,
8792
disableReactionStamps = false,
8893
onSeek,
94+
enhancedAudioUrl,
95+
enhancedAudioStatus,
8996
}: Props) {
9097
const [currentCue, setCurrentCue] = useState<string>("");
9198
const [controlsVisible, setControlsVisible] = useState(false);
@@ -104,6 +111,80 @@ export function CapVideoPlayer({
104111
const maxRetries = 3;
105112
const [duration, setDuration] = useState(0);
106113

114+
const [enhancedAudioEnabled, setEnhancedAudioEnabled] = useState(false);
115+
const enhancedAudioRef = useRef<HTMLAudioElement | null>(null);
116+
117+
const syncEnhancedAudio = useCallback(() => {
118+
if (!enhancedAudioRef.current || !videoRef.current) return;
119+
enhancedAudioRef.current.currentTime = videoRef.current.currentTime;
120+
enhancedAudioRef.current.playbackRate = videoRef.current.playbackRate;
121+
}, [videoRef]);
122+
123+
useEffect(() => {
124+
const video = videoRef.current;
125+
const audio = enhancedAudioRef.current;
126+
if (!video || !audio) return;
127+
128+
const handlePlay = () => {
129+
if (enhancedAudioEnabled) {
130+
syncEnhancedAudio();
131+
audio.play().catch(() => {});
132+
}
133+
};
134+
135+
const handlePause = () => {
136+
audio.pause();
137+
};
138+
139+
const handleSeeked = () => {
140+
if (enhancedAudioEnabled) {
141+
syncEnhancedAudio();
142+
}
143+
};
144+
145+
const handleRateChange = () => {
146+
if (enhancedAudioEnabled) {
147+
audio.playbackRate = video.playbackRate;
148+
}
149+
};
150+
151+
const handleVolumeChange = () => {
152+
audio.volume = video.volume;
153+
};
154+
155+
video.addEventListener("play", handlePlay);
156+
video.addEventListener("pause", handlePause);
157+
video.addEventListener("seeked", handleSeeked);
158+
video.addEventListener("ratechange", handleRateChange);
159+
video.addEventListener("volumechange", handleVolumeChange);
160+
161+
return () => {
162+
video.removeEventListener("play", handlePlay);
163+
video.removeEventListener("pause", handlePause);
164+
video.removeEventListener("seeked", handleSeeked);
165+
video.removeEventListener("ratechange", handleRateChange);
166+
video.removeEventListener("volumechange", handleVolumeChange);
167+
};
168+
}, [enhancedAudioEnabled, syncEnhancedAudio, videoRef]);
169+
170+
useEffect(() => {
171+
const video = videoRef.current;
172+
const audio = enhancedAudioRef.current;
173+
if (!video || !audio) return;
174+
175+
if (enhancedAudioEnabled) {
176+
video.muted = true;
177+
audio.volume = video.volume;
178+
syncEnhancedAudio();
179+
if (!video.paused) {
180+
audio.play().catch(() => {});
181+
}
182+
} else {
183+
video.muted = false;
184+
audio.pause();
185+
}
186+
}, [enhancedAudioEnabled, syncEnhancedAudio, videoRef]);
187+
107188
useEffect(() => {
108189
const checkMobile = () => {
109190
setIsMobile(window.innerWidth < 640);
@@ -750,12 +831,31 @@ export function CapVideoPlayer({
750831
toggleCaptions={toggleCaptions}
751832
/>
752833
)}
753-
<MediaPlayerSettings />
834+
<MediaPlayerEnhancedAudio
835+
enhancedAudioStatus={enhancedAudioStatus}
836+
enhancedAudioEnabled={enhancedAudioEnabled}
837+
setEnhancedAudioEnabled={setEnhancedAudioEnabled}
838+
/>
839+
<MediaPlayerSettings
840+
enhancedAudioStatus={enhancedAudioStatus}
841+
enhancedAudioEnabled={enhancedAudioEnabled}
842+
setEnhancedAudioEnabled={setEnhancedAudioEnabled}
843+
/>
754844
<MediaPlayerPiP />
755845
<MediaPlayerFullscreen />
756846
</div>
757847
</div>
758848
</MediaPlayerControls>
849+
{enhancedAudioUrl && (
850+
<audio
851+
ref={enhancedAudioRef}
852+
src={enhancedAudioUrl}
853+
preload="auto"
854+
className="hidden"
855+
>
856+
<track kind="captions" />
857+
</audio>
858+
)}
759859
</MediaPlayer>
760860
);
761861
}

‎apps/web/app/s/[videoId]/_components/HLSVideoPlayer.tsx‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import clsx from "clsx";
99
import { AnimatePresence, motion } from "framer-motion";
1010
import Hls from "hls.js";
1111
import { AlertTriangleIcon } from "lucide-react";
12-
import { useEffect, useRef, useState } from "react";
12+
import { useCallback, useEffect, useRef, useState } from "react";
1313
import { useUploadProgress } from "./ProgressCircle";
1414

1515
const { circumference } = getProgressCircleConfig();
@@ -32,6 +32,7 @@ import {
3232
MediaPlayerCaptions,
3333
MediaPlayerControls,
3434
MediaPlayerControlsOverlay,
35+
MediaPlayerEnhancedAudio,
3536
MediaPlayerError,
3637
MediaPlayerFullscreen,
3738
MediaPlayerLoading,
@@ -47,6 +48,8 @@ import {
4748
MediaPlayerVolumeIndicator,
4849
} from "./video/media-player";
4950

51+
type EnhancedAudioStatus = "PROCESSING" | "COMPLETE" | "ERROR" | "SKIPPED";
52+
5053
interface Props {
5154
videoSrc: string;
5255
videoId: Video.VideoId;
@@ -57,6 +60,8 @@ interface Props {
5760
disableCaptions?: boolean;
5861
autoplay?: boolean;
5962
hasActiveUpload?: boolean;
63+
enhancedAudioUrl?: string | null;
64+
enhancedAudioStatus?: EnhancedAudioStatus | null;
6065
}
6166

6267
export function HLSVideoPlayer({
@@ -69,6 +74,8 @@ export function HLSVideoPlayer({
6974
autoplay = false,
7075
hasActiveUpload,
7176
disableCaptions,
77+
enhancedAudioUrl,
78+
enhancedAudioStatus,
7279
}: Props) {
7380
const hlsInstance = useRef<Hls | null>(null);
7481
const [currentCue, setCurrentCue] = useState<string>("");
@@ -78,6 +85,80 @@ export function HLSVideoPlayer({
7885
const [videoLoaded, setVideoLoaded] = useState(false);
7986
const [hasPlayedOnce, setHasPlayedOnce] = useState(false);
8087

88+
const [enhancedAudioEnabled, setEnhancedAudioEnabled] = useState(false);
89+
const enhancedAudioRef = useRef<HTMLAudioElement | null>(null);
90+
91+
const syncEnhancedAudio = useCallback(() => {
92+
if (!enhancedAudioRef.current || !videoRef.current) return;
93+
enhancedAudioRef.current.currentTime = videoRef.current.currentTime;
94+
enhancedAudioRef.current.playbackRate = videoRef.current.playbackRate;
95+
}, [videoRef]);
96+
97+
useEffect(() => {
98+
const video = videoRef.current;
99+
const audio = enhancedAudioRef.current;
100+
if (!video || !audio) return;
101+
102+
const handlePlay = () => {
103+
if (enhancedAudioEnabled) {
104+
syncEnhancedAudio();
105+
audio.play().catch(() => {});
106+
}
107+
};
108+
109+
const handlePause = () => {
110+
audio.pause();
111+
};
112+
113+
const handleSeeked = () => {
114+
if (enhancedAudioEnabled) {
115+
syncEnhancedAudio();
116+
}
117+
};
118+
119+
const handleRateChange = () => {
120+
if (enhancedAudioEnabled) {
121+
audio.playbackRate = video.playbackRate;
122+
}
123+
};
124+
125+
const handleVolumeChange = () => {
126+
audio.volume = video.volume;
127+
};
128+
129+
video.addEventListener("play", handlePlay);
130+
video.addEventListener("pause", handlePause);
131+
video.addEventListener("seeked", handleSeeked);
132+
video.addEventListener("ratechange", handleRateChange);
133+
video.addEventListener("volumechange", handleVolumeChange);
134+
135+
return () => {
136+
video.removeEventListener("play", handlePlay);
137+
video.removeEventListener("pause", handlePause);
138+
video.removeEventListener("seeked", handleSeeked);
139+
video.removeEventListener("ratechange", handleRateChange);
140+
video.removeEventListener("volumechange", handleVolumeChange);
141+
};
142+
}, [enhancedAudioEnabled, syncEnhancedAudio, videoRef]);
143+
144+
useEffect(() => {
145+
const video = videoRef.current;
146+
const audio = enhancedAudioRef.current;
147+
if (!video || !audio) return;
148+
149+
if (enhancedAudioEnabled) {
150+
video.muted = true;
151+
audio.volume = video.volume;
152+
syncEnhancedAudio();
153+
if (!video.paused) {
154+
audio.play().catch(() => {});
155+
}
156+
} else {
157+
video.muted = false;
158+
audio.pause();
159+
}
160+
}, [enhancedAudioEnabled, syncEnhancedAudio, videoRef]);
161+
81162
useEffect(() => {
82163
const video = videoRef.current;
83164
if (!video) return;
@@ -468,12 +549,31 @@ export function HLSVideoPlayer({
468549
toggleCaptions={toggleCaptions}
469550
/>
470551
)}
471-
<MediaPlayerSettings />
552+
<MediaPlayerEnhancedAudio
553+
enhancedAudioStatus={enhancedAudioStatus}
554+
enhancedAudioEnabled={enhancedAudioEnabled}
555+
setEnhancedAudioEnabled={setEnhancedAudioEnabled}
556+
/>
557+
<MediaPlayerSettings
558+
enhancedAudioStatus={enhancedAudioStatus}
559+
enhancedAudioEnabled={enhancedAudioEnabled}
560+
setEnhancedAudioEnabled={setEnhancedAudioEnabled}
561+
/>
472562
<MediaPlayerPiP />
473563
<MediaPlayerFullscreen />
474564
</div>
475565
</div>
476566
</MediaPlayerControls>
567+
{enhancedAudioUrl && (
568+
<audio
569+
ref={enhancedAudioRef}
570+
src={enhancedAudioUrl}
571+
preload="auto"
572+
className="hidden"
573+
>
574+
<track kind="captions" />
575+
</audio>
576+
)}
477577
</MediaPlayer>
478578
);
479579
}

0 commit comments

Comments
 (0)