diff --git a/emcc.py b/emcc.py index 5fda8547412f8..350475013a9e4 100644 --- a/emcc.py +++ b/emcc.py @@ -2555,11 +2555,15 @@ def phase_linker_setup(options, state, newargs): if settings.AUDIO_WORKLET: if settings.AUDIO_WORKLET == 1: settings.AUDIO_WORKLET_FILE = unsuffixed(os.path.basename(target)) + '.aw.js' + if not settings.MINIMAL_RUNTIME: + # MINIMAL_RUNTIME exports these manually, since this export mechanism is placed + # in global scope that is not suitable for MINIMAL_RUNTIME loader. + settings.EXPORTED_RUNTIME_METHODS += ['stackSave', 'stackAlloc', 'stackRestore'] + elif settings.AUDIO_WORKLET == 2: + settings.BINARYEN_ASYNC_COMPILATION = 0 # Run synchronous Wasm initialization inside AudioWorklet. + settings.SINGLE_FILE = 1 # All code must be embedded in a single .js file when targeting -sAUDIO_WORKLET=2 + settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_webaudio.js'))) - if not settings.MINIMAL_RUNTIME: - # MINIMAL_RUNTIME exports these manually, since this export mechanism is placed - # in global scope that is not suitable for MINIMAL_RUNTIME loader. - settings.EXPORTED_RUNTIME_METHODS += ['stackSave', 'stackAlloc', 'stackRestore'] if settings.FORCE_FILESYSTEM and not settings.MINIMAL_RUNTIME: # when the filesystem is forced, we export by default methods that filesystem usage diff --git a/src/base64Utils.js b/src/base64Utils.js index 20ab91b154ada..14c93642966e8 100644 --- a/src/base64Utils.js +++ b/src/base64Utils.js @@ -4,7 +4,7 @@ * SPDX-License-Identifier: MIT */ -#if POLYFILL && (ENVIRONMENT_MAY_BE_SHELL || (ENVIRONMENT_MAY_BE_NODE && MIN_NODE_VERSION < 160000)) +#if POLYFILL && (ENVIRONMENT_MAY_BE_SHELL || (ENVIRONMENT_MAY_BE_NODE && MIN_NODE_VERSION < 160000) || AUDIO_WORKLET) #include "polyfill/atob.js" #endif diff --git a/src/library_webaudio.js b/src/library_webaudio.js index b37231ada4d15..97457332ba0e0 100644 --- a/src/library_webaudio.js +++ b/src/library_webaudio.js @@ -1,13 +1,38 @@ -#if AUDIO_WORKLET && !WASM_WORKERS -#error "Building with -sAUDIO_WORKLET also requires enabling -sWASM_WORKERS!" +/* +Emscripten can target Audio Worklets in two different ways: + +1) -sAUDIO_WORKLET + -sWASM_WORKERS mode: User compiles a multithreaded WebAssembly program Module, which is then + shared to the Audio Worklet scope via postMessage()ing the Module over. + Audio Worklet and main program utilize the same shared WebAssembly Memory, + and can synchronize using lock-free atomics. +2) -sAUDIO_WORKLET=2 mode: User compiles a singlethreaded WebAssembly program Module, which is loaded into Audio + Worklet scope manually by a custom web page & JS audio engine developed by the user. Main + web page does not contain any Emscripten-compiled content, no shared memory or multi- + threading is in use. +*/ + +#if AUDIO_WORKLET == 1 && !WASM_WORKERS +#error "Building with -sAUDIO_WORKLET=1 also requires enabling -sWASM_WORKERS!" #endif #if AUDIO_WORKLET && TEXTDECODER == 2 #error "-sAUDIO_WORKLET does not support -sTEXTDECODER=2 since TextDecoder is not available in AudioWorkletGlobalScope! Use e.g. -sTEXTDECODER=1 when building with -sAUDIO_WORKLET" #endif -#if AUDIO_WORKLET && SINGLE_FILE -#error "-sAUDIO_WORKLET does not support -sSINGLE_FILE" +#if AUDIO_WORKLET == 1 && SINGLE_FILE +#error "-sAUDIO_WORKLET=1 does not support -sSINGLE_FILE" #endif +#if AUDIO_WORKLET == 2 && SHARED_MEMORY +#error "Building with -sAUDIO_WORKLET=2 does not support shared memory! (-sWASM_WORKERS nor -pthread). To target Audio Worklets with shared memory, use -sAUDIO_WORKLET=1 mode." +#endif +#if AUDIO_WORKLET == 2 && BINARYEN_ASYNC_COMPILATION +#error "Building with -sAUDIO_WORKLET=2 requires -sBINARYEN_ASYNC_COMPILATION=0" +#endif +#if AUDIO_WORKLET == 2 && !SINGLE_FILE +#error "Building with -sAUDIO_WORKLET=2 requires -sSINGLE_FILE=1" +#endif + +#if AUDIO_WORKLET == 1 // None of the code below is used in -sAUDIO_WORKLET=2 mode. + let LibraryWebAudio = { $EmAudio: {}, $EmAudioCounter: 0, @@ -334,3 +359,4 @@ let LibraryWebAudio = { }; addToLibrary(LibraryWebAudio); +#endif // ~AUDIO_WORKLET==1 diff --git a/src/settings.js b/src/settings.js index 5764f26d97f8c..77df7719216aa 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1546,7 +1546,10 @@ var SHARED_MEMORY = false; // [compile+link] - affects user code at compile and system libraries at link. var WASM_WORKERS = 0; -// If true, enables targeting Wasm Web Audio AudioWorklets. Check out the +// If 1, enables targeting Wasm Web Audio AudioWorklets with Wasm Workers. +// If 2, enables building an Emscripten output Module that is compatible to be +// manually loaded inside an Audio Worklet scope. +// Check out the // full documentation in site/source/docs/api_reference/wasm_audio_worklets.rst // [link] var AUDIO_WORKLET = 0; diff --git a/test/test_interactive.py b/test/test_interactive.py index 2960c58a502db..7e1dc7ac0e9b1 100644 --- a/test/test_interactive.py +++ b/test/test_interactive.py @@ -283,3 +283,18 @@ def test_audio_worklet_tone_generator(self): # Tests that AUDIO_WORKLET+MINIMAL_RUNTIME+MODULARIZE combination works together. def test_audio_worklet_modularize(self): self.btest('webaudio/audioworklet.c', expected='0', args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-sMINIMAL_RUNTIME', '-sMODULARIZE']) + + # This test verifies -sAUDIO_WORKLET=2 mode, where Emscripten output WebAssembly Module is compiled to be manually loadable + # inside a hand-written Audio Worklet processor node. The main page will not have any Emscripten produced code in it. + @parameterized({ + 'debug': (['-g'],), # Build with verbose debug flags with assertions enabled + 'optimized': (['-Oz', '-sMINIMAL_RUNTIME'],), # Build with as optimized minimal flags as possible + }) + def test_audio_worklet_singlethreaded(self, args): + # Deploy test files from test/ source directory over to the test working directory + for f in ['main.html', 'wasm-worklet-processor.js']: + shutil.copyfile(test_file(f'webaudio/singlethreaded_audioworklet/{f}'), f) + # Build and run the test + self.compile_btest([test_file('webaudio/singlethreaded_audioworklet/SimpleKernel.cc'), '-o', 'simple-kernel.wasmmodule.js', + '--bind', '-sBINARYEN_ASYNC_COMPILATION=0', '-sSINGLE_FILE=1', '-sEXPORTED_FUNCTIONS=_malloc', '-sAUDIO_WORKLET=2', '--post-js', test_file('webaudio/singlethreaded_audioworklet/export_sync_es6_module.js')] + args) + self.run_browser('main.html', '/report_result?0') diff --git a/test/webaudio/singlethreaded_audioworklet/SimpleKernel.cc b/test/webaudio/singlethreaded_audioworklet/SimpleKernel.cc new file mode 100644 index 0000000000000..d4fc77616970b --- /dev/null +++ b/test/webaudio/singlethreaded_audioworklet/SimpleKernel.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#include +#include +#include + +const unsigned kRenderQuantumFrames = 128; +const unsigned kBytesPerChannel = kRenderQuantumFrames * sizeof(float); + +// The "kernel" is an object that processes a audio stream, which contains +// one or more channels. It is supposed to obtain the frame data from an +// |input|, process and fill an |output| of the AudioWorkletProcessor. +// +// AudioWorkletProcessor Input(multi-channel, 128-frames) +// | +// V +// Kernel +// | +// V +// AudioWorkletProcessor Output(multi-channel, 128-frames) +// +// In this implementation, the kernel operates based on 128-frames, which is +// the render quantum size of Web Audio API. +extern "C" void EMSCRIPTEN_KEEPALIVE SimpleKernel_Process(uintptr_t input_ptr, uintptr_t output_ptr, + unsigned channel_count) { + float* input_buffer = reinterpret_cast(input_ptr); + float* output_buffer = reinterpret_cast(output_ptr); + + // Bypasses the data. By design, the channel count will always be the same + // for |input_buffer| and |output_buffer|. + for (unsigned channel = 0; channel < channel_count; ++channel) { + float* destination = output_buffer + channel * kRenderQuantumFrames; + float* source = input_buffer + channel * kRenderQuantumFrames; + memcpy(destination, source, kBytesPerChannel); + } +} diff --git a/test/webaudio/singlethreaded_audioworklet/export_sync_es6_module.js b/test/webaudio/singlethreaded_audioworklet/export_sync_es6_module.js new file mode 100644 index 0000000000000..8f15b43dcee98 --- /dev/null +++ b/test/webaudio/singlethreaded_audioworklet/export_sync_es6_module.js @@ -0,0 +1,6 @@ +// Emscripten has a -sEXPORT_ES6=1 option to export the generated program as a 'Module' +// object. However, that module is asynchronously loaded, which does not work well +// for Audio Worklets, which require synchronous loading. So instead we manually +// build the page without EXPORT_ES6, and just add an export here. + +export default Module; diff --git a/test/webaudio/singlethreaded_audioworklet/main.html b/test/webaudio/singlethreaded_audioworklet/main.html new file mode 100644 index 0000000000000..a270521d0ec07 --- /dev/null +++ b/test/webaudio/singlethreaded_audioworklet/main.html @@ -0,0 +1,60 @@ + + + + + + Emscripten-Generated Code + + +

Audio Worklet and WebAssembly

+

A basic pattern to use Audio Worklet with WebAssembly. The + AudioWorkletProcessor simply bypasses (copies) the audio via the WebAssembly + function.

+

See + Chrome Developers Article: Audio Worklet Design Pattern + for more details.

+ +
+ +
+ + + + diff --git a/test/webaudio/singlethreaded_audioworklet/wasm-worklet-processor.js b/test/webaudio/singlethreaded_audioworklet/wasm-worklet-processor.js new file mode 100644 index 0000000000000..ab685db4f0d4f --- /dev/null +++ b/test/webaudio/singlethreaded_audioworklet/wasm-worklet-processor.js @@ -0,0 +1,342 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import Module from './simple-kernel.wasmmodule.js'; + +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +// Byte per audio sample. (32 bit float) +const BYTES_PER_SAMPLE = Float32Array.BYTES_PER_ELEMENT; + +// Basic byte unit of WASM heap. (16 bit = 2 bytes) +const BYTES_PER_UNIT = Uint16Array.BYTES_PER_ELEMENT; + +// The max audio channel on Chrome is 32. +const MAX_CHANNEL_COUNT = 32; + +// WebAudio's render quantum size. +const RENDER_QUANTUM_FRAMES = 128; + + +/** + * A WASM HEAP wrapper for AudioBuffer class. This breaks down the AudioBuffer + * into an Array of Float32Array for the convinient WASM opearion. + * + * @class + * @dependency Module A WASM module generated by the emscripten glue code. + */ +class HeapAudioBuffer { + /** + * @constructor + * @param {object} wasmModule WASM module generated by Emscripten. + * @param {number} length Buffer frame length. + * @param {number} channelCount Number of channels. + * @param {number=} maxChannelCount Maximum number of channels. + */ + constructor(wasmModule, length, channelCount, maxChannelCount) { + // The |channelCount| must be greater than 0, and less than or equal to + // the maximum channel count. + this._isInitialized = false; + this._module = wasmModule; + this._length = length; + this._maxChannelCount = maxChannelCount ? + Math.min(maxChannelCount, MAX_CHANNEL_COUNT) : channelCount; + this._channelCount = channelCount; + this._allocateHeap(); + this._isInitialized = true; + } + + /** + * Allocates memory in the WASM heap and set up Float32Array views for the + * channel data. + * + * @private + */ + _allocateHeap() { + const channelByteSize = this._length * BYTES_PER_SAMPLE; + const dataByteSize = this._channelCount * channelByteSize; + this._dataPtr = this._module._malloc(dataByteSize); + this._channelData = []; + for (let i = 0; i < this._channelCount; ++i) { + const startByteOffset = this._dataPtr + i * channelByteSize; + const endByteOffset = startByteOffset + channelByteSize; + // Get the actual array index by dividing the byte offset by 2 bytes. + this._channelData[i] = + this._module.HEAPF32.subarray( + startByteOffset >> BYTES_PER_UNIT, + endByteOffset >> BYTES_PER_UNIT); + } + } + + /** + * Adapt the current channel count to the new input buffer. + * + * @param {number} newChannelCount The new channel count. + */ + adaptChannel(newChannelCount) { + if (newChannelCount < this._maxChannelCount) { + this._channelCount = newChannelCount; + } + } + + /** + * Getter for the buffer length in frames. + * + * @return {?number} Buffer length in frames. + */ + get length() { + return this._isInitialized ? this._length : null; + } + + /** + * Getter for the number of channels. + * + * @return {?number} Buffer length in frames. + */ + get numberOfChannels() { + return this._isInitialized ? this._channelCount : null; + } + + /** + * Getter for the maxixmum number of channels allowed for the instance. + * + * @return {?number} Buffer length in frames. + */ + get maxChannelCount() { + return this._isInitialized ? this._maxChannelCount : null; + } + + /** + * Returns a Float32Array object for a given channel index. If the channel + * index is undefined, it returns the reference to the entire array of channel + * data. + * + * @param {number|undefined} channelIndex Channel index. + * @return {?Array} a channel data array or an + * array of channel data. + */ + getChannelData(channelIndex) { + if (channelIndex >= this._channelCount) { + return null; + } + + return typeof channelIndex === 'undefined' ? + this._channelData : this._channelData[channelIndex]; + } + + /** + * Returns the base address of the allocated memory space in the WASM heap. + * + * @return {number} WASM Heap address. + */ + getHeapAddress() { + return this._dataPtr; + } + + /** + * Returns the base address of the allocated memory space in the WASM heap. + * + * @return {number} WASM Heap address. + */ + getPointer() { + return this._dataPtr; + } + + /** + * Frees the allocated memory space in the WASM heap. + */ + free() { + this._isInitialized = false; + this._module._free(this._dataPtr); + this._module._free(this._pointerArrayPtr); + this._channelData = null; + } +} // class HeapAudioBuffer + + +/** + * A JS FIFO implementation for the AudioWorklet. 3 assumptions for the + * simpler operation: + * 1. the push and the pull operation are done by 128 frames. (Web Audio + * API's render quantum size in the speficiation) + * 2. the channel count of input/output cannot be changed dynamically. + * The AudioWorkletNode should be configured with the `.channelCount = k` + * (where k is the channel count you want) and + * `.channelCountMode = explicit`. + * 3. This is for the single-thread operation. (obviously) + * + * @class + */ +class RingBuffer { + /** + * @constructor + * @param {number} length Buffer length in frames. + * @param {number} channelCount Buffer channel count. + */ + constructor(length, channelCount) { + this._readIndex = 0; + this._writeIndex = 0; + this._framesAvailable = 0; + + this._channelCount = channelCount; + this._length = length; + this._channelData = []; + for (let i = 0; i < this._channelCount; ++i) { + this._channelData[i] = new Float32Array(length); + } + } + + /** + * Getter for Available frames in buffer. + * + * @return {number} Available frames in buffer. + */ + get framesAvailable() { + return this._framesAvailable; + } + + /** + * Push a sequence of Float32Arrays to buffer. + * + * @param {array} arraySequence A sequence of Float32Arrays. + */ + push(arraySequence) { + // The channel count of arraySequence and the length of each channel must + // match with this buffer obejct. + + // Transfer data from the |arraySequence| storage to the internal buffer. + const sourceLength = arraySequence[0].length; + for (let i = 0; i < sourceLength; ++i) { + const writeIndex = (this._writeIndex + i) % this._length; + for (let channel = 0; channel < this._channelCount; ++channel) { + this._channelData[channel][writeIndex] = arraySequence[channel][i]; + } + } + + this._writeIndex = (this._writeIndex + sourceLength) % this._length; + + // For excessive frames, the buffer will be overwritten. + this._framesAvailable += sourceLength; + if (this._framesAvailable > this._length) { + this._framesAvailable = this._length; + } + } + + /** + * Pull data out of buffer and fill a given sequence of Float32Arrays. + * + * @param {array} arraySequence An array of Float32Arrays. + */ + pull(arraySequence) { + // The channel count of arraySequence and the length of each channel must + // match with this buffer obejct. + + // If the FIFO is completely empty, do nothing. + if (this._framesAvailable === 0) { + return; + } + + const destinationLength = arraySequence[0].length; + + // Transfer data from the internal buffer to the |arraySequence| storage. + for (let i = 0; i < destinationLength; ++i) { + const readIndex = (this._readIndex + i) % this._length; + for (let channel = 0; channel < this._channelCount; ++channel) { + arraySequence[channel][i] = this._channelData[channel][readIndex]; + } + } + + this._readIndex = (this._readIndex + destinationLength) % this._length; + + this._framesAvailable -= destinationLength; + if (this._framesAvailable < 0) { + this._framesAvailable = 0; + } + } +} // class RingBuffer + + +export { + MAX_CHANNEL_COUNT, + RENDER_QUANTUM_FRAMES, + HeapAudioBuffer, + RingBuffer, +}; + +/** + * A simple demonstration of WASM-powered AudioWorkletProcessor. + * + * @class WASMWorkletProcessor + * @extends AudioWorkletProcessor + */ +class WASMWorkletProcessor extends AudioWorkletProcessor { + /** + * @constructor + */ + constructor() { + super(); + + // Allocate the buffer for the heap access. Start with stereo, but it can + // be expanded up to 32 channels. + this._heapInputBuffer = new HeapAudioBuffer( + Module, RENDER_QUANTUM_FRAMES, 2, MAX_CHANNEL_COUNT); + this._heapOutputBuffer = new HeapAudioBuffer( + Module, RENDER_QUANTUM_FRAMES, 2, MAX_CHANNEL_COUNT); + this._kernel_process = Module._SimpleKernel_Process; + } + + /** + * System-invoked process callback function. + * @param {Array} inputs Incoming audio stream. + * @param {Array} outputs Outgoing audio stream. + * @param {Object} parameters AudioParam data. + * @return {Boolean} Active source flag. + */ + process(inputs, outputs, parameters) { + // Use the 1st input and output only to make the example simpler. |input| + // and |output| here have the similar structure with the AudioBuffer + // interface. (i.e. An array of Float32Array) + const input = inputs[0]; + const output = outputs[0]; + + // For this given render quantum, the channel count of the node is fixed + // and identical for the input and the output. + const channelCount = input.length; + + // Prepare HeapAudioBuffer for the channel count change in the current + // render quantum. + this._heapInputBuffer.adaptChannel(channelCount); + this._heapOutputBuffer.adaptChannel(channelCount); + + // Copy-in, process and copy-out. + for (let channel = 0; channel < channelCount; ++channel) { + this._heapInputBuffer.getChannelData(channel).set(input[channel]); + } + this._kernel_process( + this._heapInputBuffer.getHeapAddress(), + this._heapOutputBuffer.getHeapAddress(), + channelCount); + for (let channel = 0; channel < channelCount; ++channel) { + output[channel].set(this._heapOutputBuffer.getChannelData(channel)); + } + + return true; + } +} + +registerProcessor('wasm-worklet-processor', WASMWorkletProcessor); diff --git a/tools/building.py b/tools/building.py index e2485e9ee54cc..723a4ca227e9a 100644 --- a/tools/building.py +++ b/tools/building.py @@ -361,6 +361,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False): cmd += ['--exportES6'] if settings.VERBOSE: cmd += ['verbose'] + cmd += ['--exportES6'] # XXXXXXXXXX figure out how to pass this without passing EXPORT_ES6 maybe? if return_output: return check_call(cmd, stdout=PIPE).stdout