examples/webgpu_audio_processing.html

<html lang="en">
	<head>
		<title>three.js - WebGPU - Audio Processing</title>
		<meta charset="utf-8">
		<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
		<link type="text/css" rel="stylesheet" href="main.css">
	</head>
	<body>

		<div id="overlay">
			<button id="startButton">Play</button>
		</div>

		<div id="info">
			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> WebGPU - Audio Processing
			<br>Click on the screen to process the audio using WebGPU.
		</div>

		<script async src="https://unpkg.com/es-module-shims@1.3.6/dist/es-module-shims.js"></script>

		<script type="importmap">
			{
				"imports": {
					"three": "../build/three.module.js",
					"three/addons/": "./jsm/",
					"three/nodes": "./jsm/nodes/Nodes.js"
				}
			}
		</script>

		<script type="module">

			import * as THREE from 'three';

			import {
				ShaderNode, compute,
				uniform, element, storage, instanceIndex,
				float, assign, add, sub, div, mul, texture, viewportTopLeft, color
			} from 'three/nodes';

			import { GUI } from 'three/addons/libs/lil-gui.module.min.js';

			import WebGPU from 'three/addons/capabilities/WebGPU.js';
			import WebGPURenderer from 'three/addons/renderers/webgpu/WebGPURenderer.js';

			let camera, scene, renderer;
			let computeNode;
			let waveBuffer, sampleRate;
			let waveGPUBuffer;
			let currentAudio, currentAnalyser;
			let analyserBuffer = new Uint8Array( 1024 );
			let analyserTexture;

			await init();

			async function playAudioBuffer() {

				if ( currentAudio ) currentAudio.stop();

				// compute audio

				renderer.compute( computeNode );

				const waveArray = await renderer.getArrayFromBuffer( waveGPUBuffer );

				// play result

				const audioOutputContext = new AudioContext( { sampleRate } );
				const audioOutputBuffer = audioOutputContext.createBuffer( 1, waveArray.length, sampleRate );

				audioOutputBuffer.copyToChannel( waveArray, 0 );

				const source = audioOutputContext.createBufferSource();
				source.connect( audioOutputContext.destination );
				source.buffer = audioOutputBuffer;
				source.start();

				currentAudio = source;

				// visual feedback

				currentAnalyser = audioOutputContext.createAnalyser();
				currentAnalyser.fftSize = 2048;

				source.connect( currentAnalyser );

			}

			async function init() {

				if ( WebGPU.isAvailable() === false ) {

					document.body.appendChild( WebGPU.getErrorMessage() );

					throw new Error( 'No WebGPU support' );

				}

				document.onclick = () => {

					const overlay = document.getElementById( 'overlay' );
					if ( overlay !== null ) overlay.remove();

					playAudioBuffer();

				};

				// audio buffer

				const soundBuffer = await fetch( 'sounds/webgpu-audio-processing.mp3' ).then( res => res.arrayBuffer() );
				const audioContext = new AudioContext();

				const audioBuffer = await audioContext.decodeAudioData( soundBuffer );

				waveBuffer = audioBuffer.getChannelData( 0 );

				// adding extra silence to delay and pitch
				waveBuffer = new Float32Array( [ ...waveBuffer, ...new Float32Array( 200000 ) ] );

				sampleRate = audioBuffer.sampleRate / audioBuffer.numberOfChannels;


				// create webgpu buffers

				waveGPUBuffer = new THREE.InstancedBufferAttribute( waveBuffer, 1 );

				const waveStorageNode = storage( waveGPUBuffer, 'float', waveBuffer.length );


				// read-only buffer

				const waveNode = storage( new THREE.InstancedBufferAttribute( waveBuffer, 1 ), 'float', waveBuffer.length );


				// params

				const pitch = uniform( 1.5 );
				const delayVolume = uniform( .2 );
				const delayOffset = uniform( .55 );


				// compute (shader-node)

				const computeShaderNode = new ShaderNode( ( inputs, builder ) => {

					const index = float( instanceIndex );


					// pitch

					const time = mul( index, pitch );

					let wave = element( waveNode, time );


					// delay

					for ( let i = 1; i < 7; i ++ ) {

						const waveOffset = element( waveNode, mul( sub( index, mul( mul( delayOffset, sampleRate ), i ) ), pitch ) );
						const waveOffsetVolume = mul( waveOffset, div( delayVolume, i * i ) );

						wave = add( wave, waveOffsetVolume );

					}


					// store

					const waveStorageElementNode = element( waveStorageNode, instanceIndex );

					assign( waveStorageElementNode, wave ).build( builder );

				} );


				// compute

				computeNode = compute( computeShaderNode, waveBuffer.length );


				// gui

				const gui = new GUI();

				gui.add( pitch, 'value', .5, 2, 0.01 ).name( 'pitch' );
				gui.add( delayVolume, 'value', 0, 1, .01 ).name( 'delayVolume' );
				gui.add( delayOffset, 'value', .1, 1, .01 ).name( 'delayOffset' );


				// renderer

				const container = document.createElement( 'div' );
				document.body.appendChild( container );

				camera = new THREE.PerspectiveCamera( 45, window.innerWidth / window.innerHeight, 0.01, 30 );


				// nodes

				analyserTexture = new THREE.DataTexture( analyserBuffer, analyserBuffer.length, 1, THREE.RedFormat );

				const spectrum = mul( texture( analyserTexture, viewportTopLeft.x ).x, viewportTopLeft.y );
				const backgroundNode = mul( color( 0x0000FF ), spectrum );


				// scene

				scene = new THREE.Scene();
				scene.backgroundNode = backgroundNode;

				// renderer

				renderer = new WebGPURenderer();
				renderer.setPixelRatio( window.devicePixelRatio );
				renderer.setSize( window.innerWidth, window.innerHeight );
				renderer.setAnimationLoop( render );
				container.appendChild( renderer.domElement );

				window.addEventListener( 'resize', onWindowResize );

			}

			function onWindowResize() {

				camera.aspect = window.innerWidth / window.innerHeight;
				camera.updateProjectionMatrix();

				renderer.setSize( window.innerWidth, window.innerHeight );

			}

			function render() {

				if ( currentAnalyser ) {

					currentAnalyser.getByteFrequencyData( analyserBuffer );

					analyserTexture.needsUpdate = true;

				}

				renderer.render( scene, camera );

			}

		</script>
	</body>
</html>