Merge pull request #4666 from remotion-dev/respect-audio-presentation…

…-offset
remotion-dev · Dec 24, 2024 · 08c5668 · 08c5668
2 parents 7207c80 + 1f71004
commit 08c5668
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 8 deletions.
diff --git a/packages/renderer/src/assets/download-map.ts b/packages/renderer/src/assets/download-map.ts
@@ -9,6 +9,7 @@ import type {RenderMediaOnDownload} from './download-and-map-assets-to-file';
 export type AudioChannelsAndDurationResultCache = {
 	channels: number;
 	duration: number | null;
+	startTime: number | null;
 };
 
 export type DownloadMap = {

diff --git a/packages/renderer/src/assets/get-audio-channels.ts b/packages/renderer/src/assets/get-audio-channels.ts
@@ -24,7 +24,8 @@ export const getAudioChannelsAndDurationWithoutCache = async ({
 }) => {
 	const args = [
 		['-v', 'error'],
-		['-show_entries', 'stream=channels:format=duration'],
+		['-select_streams', 'a:0'],
+		['-show_entries', 'stream=channels:stream=start_time:format=duration'],
 		['-of', 'default=nw=1'],
 		[src],
 	]
@@ -42,10 +43,12 @@ export const getAudioChannelsAndDurationWithoutCache = async ({
 		});
 		const channels = task.stdout.match(/channels=([0-9]+)/);
 		const duration = task.stdout.match(/duration=([0-9.]+)/);
+		const startTime = task.stdout.match(/start_time=([0-9.]+)/);
 
 		const result: AudioChannelsAndDurationResultCache = {
 			channels: channels ? parseInt(channels[1], 10) : 0,
 			duration: duration ? parseFloat(duration[1]) : null,
+			startTime: startTime ? parseFloat(startTime[1]) : null,
 		};
 		return result;
 	} catch (err) {

diff --git a/packages/renderer/src/preprocess-audio-track.ts b/packages/renderer/src/preprocess-audio-track.ts
@@ -50,7 +50,7 @@ const preprocessAudioTrackUnlimited = async ({
 	trimRightOffset,
 	forSeamlessAacConcatenation,
 }: Options): Promise<PreprocessedAudioTrack | null> => {
-	const {channels, duration} = await getAudioChannelsAndDuration({
+	const {channels, duration, startTime} = await getAudioChannelsAndDuration({
 		downloadMap,
 		src: resolveAssetSrc(asset.src),
 		indent,
@@ -71,6 +71,7 @@ const preprocessAudioTrackUnlimited = async ({
 		volume: flattenVolumeArray(asset.volume),
 		indent,
 		logLevel,
+		presentationTimeOffsetInSeconds: startTime ?? 0,
 	});
 
 	if (filter === null) {
@@ -96,7 +97,7 @@ const preprocessAudioTrackUnlimited = async ({
 		'Filter:',
 		filter.filter,
 	);
-	const startTime = Date.now();
+	const startTimestamp = Date.now();
 
 	const task = callFf({
 		bin: 'ffmpeg',
@@ -122,7 +123,7 @@ const preprocessAudioTrackUnlimited = async ({
 	Log.verbose(
 		{indent, logLevel},
 		'Preprocessed audio track',
-		`${Date.now() - startTime}ms`,
+		`${Date.now() - startTimestamp}ms`,
 	);
 
 	cleanup();

diff --git a/packages/renderer/src/stringify-ffmpeg-filter.ts b/packages/renderer/src/stringify-ffmpeg-filter.ts
@@ -179,6 +179,7 @@ export const stringifyFfmpegFilter = ({
 	asset,
 	indent,
 	logLevel,
+	presentationTimeOffsetInSeconds,
 }: {
 	channels: number;
 	volume: AssetVolume;
@@ -191,6 +192,7 @@ export const stringifyFfmpegFilter = ({
 	asset: MediaAsset;
 	indent: boolean;
 	logLevel: LogLevel;
+	presentationTimeOffsetInSeconds: number;
 }): FilterWithoutPaddingApplied | null => {
 	if (channels === 0) {
 		return null;
@@ -242,6 +244,8 @@ export const stringifyFfmpegFilter = ({
 
 	const padAtEnd = chunkLengthInSeconds - audibleDuration - startInVideoSeconds;
 
+	const padStart = startInVideoSeconds + presentationTimeOffsetInSeconds;
+
 	// Set as few filters as possible, as combining them can create noise
 	return {
 		filter:
@@ -273,10 +277,10 @@ export const stringifyFfmpegFilter = ({
 			// This should be fine because FFMPEG documentation states:
 			// "Unused delays will be silently ignored."
 			// https://ffmpeg.org/ffmpeg-filters.html#adelay
-			startInVideoSeconds === 0
+			padStart === 0
 				? null
 				: `adelay=${new Array(channels + 1)
-						.fill((startInVideoSeconds * 1000).toFixed(0))
+						.fill((padStart * 1000).toFixed(0))
 						.join('|')}`,
 		actualTrimLeft,
 	};

diff --git a/packages/renderer/src/test/ffmpeg-filters.test.ts b/packages/renderer/src/test/ffmpeg-filters.test.ts
@@ -51,6 +51,7 @@ test('Should create a basic filter correctly', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0,
@@ -99,6 +100,7 @@ test('Trim the end', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0,
@@ -148,6 +150,7 @@ test('Should handle trim correctly', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0.3333333333333333,
@@ -184,6 +187,7 @@ test('Should add padding if audio is too short', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0.3333333333333333,
@@ -231,6 +235,7 @@ test('Should handle delay correctly', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0.3333333333333333,
@@ -278,6 +283,7 @@ test('Should offset multiple channels', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0.3333333333333333,
@@ -337,6 +343,7 @@ test('Should calculate pad correctly with a lot of playbackRate', () => {
 			volume: flattenVolumeArray(baseAsset.volume),
 			indent: false,
 			logLevel: 'info',
+			presentationTimeOffsetInSeconds: 0,
 		}),
 	).toEqual({
 		actualTrimLeft: 0,

diff --git a/packages/renderer/src/test/get-audio-channels.test.ts b/packages/renderer/src/test/get-audio-channels.test.ts
@@ -26,7 +26,7 @@ test('Get audio channels for video', async () => {
 		binariesDirectory: null,
 		cancelSignal: undefined,
 	});
-	expect(channels).toEqual({channels: 2, duration: 10});
+	expect(channels).toEqual({channels: 2, duration: 10, startTime: 0});
 }, 90000);
 
 test('Get audio channels for video without music', async () => {
@@ -76,7 +76,7 @@ test('Get audio channels for video with music', async () => {
 	});
 	cleanDownloadMap(downloadMap);
 
-	expect(channels).toEqual({channels: 2, duration: 56.529});
+	expect(channels).toEqual({channels: 2, duration: 56.529, startTime: 0});
 }, 90000);
 
 test('Throw error if parsing a non video file', () => {