@remotion/media-utils: Add visualizeAudioWaveform() and createSmoothSvgPath() APIs

packages/docs/components/AudioWaveformExamples.tsx

@@ -0,0 +1,190 @@
+import {
+	createSmoothSvgPath,
+	useAudioData,
+	visualizeAudioWaveform,
+} from '@remotion/media-utils';
+import {Player} from '@remotion/player';
+import React from 'react';
+import {
+	AbsoluteFill,
+	Audio,
+	staticFile,
+	useCurrentFrame,
+	useVideoConfig,
+} from 'remotion';
+
+const src = staticFile('podcast.wav');
+
+const BaseExample: React.FC = () => {
+	const frame = useCurrentFrame();
+	const {fps} = useVideoConfig();
+	const audioDataVoice = useAudioData(src);
+	const {width} = useVideoConfig();
+	const height = 300;
+
+	if (!audioDataVoice) {
+		return null;
+	}
+
+	const waveform = visualizeAudioWaveform({
+		fps,
+		frame,
+		audioData: audioDataVoice,
+		numberOfSamples: 32,
+		windowInSeconds: 1 / fps,
+		channel: 0,
+	});
+
+	const p = createSmoothSvgPath({
+		points: waveform.map((x, i) => {
+			return {
+				x: (i / (waveform.length - 1)) * width,
+				y: x * height * 2 + height / 2,
+			};
+		}),
+	});
+
+	return (
+		<div style={{flex: 1}}>
+			<Audio src={src} />
+			<AbsoluteFill style={{justifyContent: 'center', alignItems: 'center'}}>
+				<svg
+					style={{backgroundColor: '#0B84F3'}}
+					viewBox={`0 0 ${width} ${height}`}
+					width={width}
+					height={height}
+				>
+					<path stroke="white" fill="none" strokeWidth={10} d={p as string} />
+				</svg>
+			</AbsoluteFill>
+		</div>
+	);
+};
+
+const MovingExample: React.FC = () => {
+	const frame = useCurrentFrame();
+	const {fps} = useVideoConfig();
+	const audioDataVoice = useAudioData(src);
+	const {width} = useVideoConfig();
+	const height = 300;
+
+	if (!audioDataVoice) {
+		return null;
+	}
+
+	const waveform = visualizeAudioWaveform({
+		fps,
+		frame,
+		audioData: audioDataVoice,
+		numberOfSamples: 32,
+		windowInSeconds: 10 / fps,
+		channel: 0,
+	});
+
+	const p = createSmoothSvgPath({
+		points: waveform.map((x, i) => {
+			return {
+				x: (i / (waveform.length - 1)) * width,
+				y: x * height * 2 + height / 2,
+			};
+		}),
+	});
+
+	return (
+		<div style={{flex: 1}}>
+			<Audio src={src} />
+			<AbsoluteFill style={{justifyContent: 'center', alignItems: 'center'}}>
+				<svg
+					style={{backgroundColor: '#0B84F3'}}
+					viewBox={`0 0 ${width} ${height}`}
+					width={width}
+					height={height}
+				>
+					<path stroke="#ffffff" fill="none" strokeWidth={10} d={p as string} />
+				</svg>
+			</AbsoluteFill>
+		</div>
+	);
+};
+
+const PosterizedExample: React.FC = () => {
+	const frame = useCurrentFrame();
+	const {fps} = useVideoConfig();
+	const audioDataVoice = useAudioData(src);
+	const {width} = useVideoConfig();
+	const height = 300;
+
+	if (!audioDataVoice) {
+		return null;
+	}
+
+	const waveform = visualizeAudioWaveform({
+		fps,
+		frame: Math.round(frame / 3) * 3,
+		audioData: audioDataVoice,
+		numberOfSamples: 16,
+		windowInSeconds: 1 / fps,
+		channel: 0,
+	});
+
+	const p = createSmoothSvgPath({
+		points: waveform.map((x, i) => {
+			return {
+				x: (i / (waveform.length - 1)) * width,
+				y: x * height * 2 + height / 2,
+			};
+		}),
+	});
+
+	return (
+		<div style={{flex: 1}}>
+			<Audio src={src} />
+			<AbsoluteFill style={{justifyContent: 'center', alignItems: 'center'}}>
+				<svg
+					style={{backgroundColor: ' #0B84F3'}}
+					viewBox={`0 0 ${width} ${height}`}
+					width={width}
+					height={height}
+				>
+					<path stroke="white" fill="none" strokeWidth={10} d={p as string} />
+				</svg>
+			</AbsoluteFill>
+		</div>
+	);
+};
+
+export const AudioWaveFormExample: React.FC<{
+	readonly type: 'base' | 'moving' | 'posterized';
+}> = ({type}) => {
+	const component = (() => {
+		if (type === 'base') {
+			return BaseExample;
+		}
+
+		if (type === 'moving') {
+			return MovingExample;
+		}
+
+		if (type === 'posterized') {
+			return PosterizedExample;
+		}
+
+		throw new TypeError('oops');
+	})();
+	return (
+		<div>
+			<Player
+				component={component}
+				compositionWidth={1080}
+				compositionHeight={300}
+				controls
+				durationInFrames={300}
+				fps={30}
+				style={{
+					width: '100%',
+				}}
+				loop
+			/>
+		</div>
+	);
+};

packages/docs/components/TableOfContents/media-utils.tsx

@@ -41,7 +41,15 @@ export const TableOfContents: React.FC = () => {
 				</TOCItem>
 				<TOCItem link="/docs/visualize-audio">
 					<strong>visualizeAudio()</strong>
-					<div>Processes a waveform for visualization</div>
+					<div>Process a music waveform for visualization</div>
+				</TOCItem>
+				<TOCItem link="/docs/media-utils/visualize-audio-waveform">
+					<strong>visualizeAudioWaveform()</strong>
+					<div>Process a voice waveform for visualization</div>
+				</TOCItem>
+				<TOCItem link="/docs/media-utils/create-smooth-svg-path">
+					<strong>createSmoothSvgPath()</strong>
+					<div>Turn waveform points into a smooth SVG path</div>
 				</TOCItem>
 			</Grid>
 		</div>

packages/docs/docs/audio-visualization.mdx

@@ -9,9 +9,11 @@ Remotion has APIs for visualizing audio, for example for creating audiograms or
 
 The `@remotion/media-utils` package provides helper functions for reading and processing audio. Using the [`getAudioData()`](/docs/get-audio-data) API you can read audio, and using the [`useAudioData()`](/docs/use-audio-data) helper hook you can load this audio data directly into your component.
 
-Using the [`visualizeAudio()`](/docs/visualize-audio) API, you can get an audio spectrum for the current frame, with the `numberOfSamples` parameter being an option to control the amount of detail you need.
+## Bar visualization
 
-Refer to the documentation of the above mentioned functions to learn more.
+Using the [`visualizeAudio()`](/docs/visualize-audio) API, you can get an audio spectrum for the current frame.
+
+Bar visualizations are ideal for visualizing music.
 
 ```tsx twoslash
 import {useAudioData, visualizeAudio} from '@remotion/media-utils';
@@ -41,21 +43,27 @@ export const MyComponent: React.FC = () => {
     <div>
       <Audio src={music} />
       {visualization.map((v) => {
-        return (
-          <div style={{width: 1000 * v, height: 15, backgroundColor: 'blue'}} />
-        );
+        return <div style={{width: 1000 * v, height: 15, backgroundColor: 'blue'}} />;
       })}
     </div>
   );
 };
 ```
 
+## Waveform visualization
+
+See an example for a waveform visualizations using [`visualizeAudioWaveform()`](/docs/media-utils/visualize-audio-waveform) here.
+
+import {AudioWaveFormExample} from '../components/AudioWaveformExamples.tsx';
+
+<AudioWaveFormExample type="moving" />
+
 ## Working with large files
 
-[`useAudioData()`](/docs/use-audio-data) loads the entire audio file into memory.  
+[`useAudioData()`](/docs/use-audio-data) loads the entire audio file into memory.
 This is fine for small files, but for large files, it can be slow and consume a lot of memory.
 
-Use [`useWindowedAudioData()`](/docs/use-windowed-audio-data) to only load a portion of the audio around the current frame.  
+Use [`useWindowedAudioData()`](/docs/use-windowed-audio-data) to only load a portion of the audio around the current frame.
 The tradeoff is that this API only works with `.wav` files.
 
 ## See also
@@ -64,3 +72,4 @@ The tradeoff is that this API only works with `.wav` files.
 - [`useAudioData()`](/docs/use-audio-data)
 - [`useWindowedAudioData()`](/docs/use-windowed-audio-data)
 - [`visualizeAudio()`](/docs/visualize-audio)
+- [`visualizeAudioWaveform()`](/docs/media-utils/visualize-audio-waveform)

packages/docs/docs/get-waveform-portion.mdx

@@ -37,6 +37,18 @@ _number_
 
 How big you want the result array to be. The function will compress the waveform to fit in `numberOfSamples` data points.
 
+### `channel`
+
+_number_
+
+Which channel to use. Defaults to 0.
+
+### `outputRange`
+
+_number_
+
+The range of the output values. Either `minus-one-to-one` or `zero-to-one`. Defaults to `zero-to-one`.
+
 ## Return value
 
 `Bar[]` - An array of objects with the following properties:

packages/docs/docs/media-utils/create-smooth-svg-path.mdx

@@ -0,0 +1,32 @@
+---
+title: createSmoothSvgPath()
+id: create-smooth-svg-path
+crumb: '@remotion/media-utils'
+---
+
+_Part of the `@remotion/media-utils`_ package of helper functions.
+
+This function takes points, usually generated from [`visualizeAudioWaveform()`](/docs/media-utils/visualize-audio-waveform) or [`visualizeAudio()`](/docs/visualize-audio), and adds SVG `C` (curve) statements inbetween them to create a smooth path.
+
+## Example
+
+```tsx twoslash
+import {createSmoothSvgPath} from '@remotion/media-utils';
+import React from 'react';
+
+const points = [
+  {x: 0, y: 0},
+  {x: 100, y: 100},
+  {x: 200, y: 50},
+  {x: 300, y: 150},
+];
+
+const path = createSmoothSvgPath({points});
+```
+
+See a practical example [here](http://localhost:3001/docs/media-utils/visualize-audio-waveform#examples).
+
+## See also
+
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/media-utils/src/create-smooth-svg-path.ts)
+- [Audio visualization](/docs/audio-visualization)