Resample to audio context rate for playback

For now we do this only for samples Web Audio can't cope with. I suspect
we need to reduce the range that we defer to Web Audio for FF and old
Safari.

This works in Chrome but nowhere else, which is a shame because
everywhere else needs it more urgently than Chrome... Puzzling over
this.

Author: microbit-matt-hillsdon

Diff for: src/board/audio/index.ts

@@ -1,3 +1,4 @@
+import { SRC } from "@alexanderolsen/libsamplerate-js/dist/src";
 import { replaceBuiltinSound } from "./built-in-sounds";
 import { SoundEmojiSynthesizer } from "./sound-emoji-synthesizer";
 import { parseSoundEffects } from "./sound-expressions";
@@ -15,7 +16,10 @@ declare global {
 
 interface AudioOptions {
   defaultAudioCallback: () => void;
+  defaultResampler: SRC;
   speechAudioCallback: () => void;
+  speechResampler: SRC;
+  soundExpressionResampler: SRC;
 }
 
 export class BoardAudio {
@@ -38,7 +42,10 @@ export class BoardAudio {
 
   initializeCallbacks({
     defaultAudioCallback,
+    defaultResampler,
     speechAudioCallback,
+    speechResampler,
+    soundExpressionResampler,
   }: AudioOptions) {
     if (!this.context) {
       throw new Error("Context must be pre-created from a user event");
@@ -60,16 +67,19 @@ export class BoardAudio {
     this.default = new BufferedAudio(
       this.context,
       this.volumeNode,
+      defaultResampler,
       defaultAudioCallback
     );
     this.speech = new BufferedAudio(
       this.context,
       this.volumeNode,
+      speechResampler,
       speechAudioCallback
     );
     this.soundExpression = new BufferedAudio(
       this.context,
       this.volumeNode,
+      soundExpressionResampler,
       () => {
         if (this.currentSoundExpressionCallback) {
           this.currentSoundExpressionCallback();
@@ -79,12 +89,13 @@ export class BoardAudio {
   }
 
   async createAudioContextFromUserInteraction(): Promise<void> {
+    // If we set a 44.1kHz rate then we fail to connect to user media on Mac as it selects 48000
+    // So we leave it at the default hoping it's most likely to match user media...
+    // Until there's progress on this there doesn't seem a better way:
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1674892
     this.context =
-      this.context ??
-      new (window.AudioContext || window.webkitAudioContext)({
-        // The highest rate is the sound expression synth.
-        sampleRate: 44100,
-      });
+      this.context ?? new (window.AudioContext || window.webkitAudioContext)();
+
     if (this.context.state === "suspended") {
       return this.context.resume();
     }
@@ -96,21 +107,16 @@ export class BoardAudio {
       this.stopSoundExpression();
     };
     const synth = new SoundEmojiSynthesizer(0, onDone);
+    this.soundExpression!.setSampleRate(synth.sampleRate);
     synth.play(soundEffects);
 
     const callback = () => {
       const source = synth.pull();
       if (this.context) {
-        // Use createBuffer instead of new AudioBuffer to support Safari 14.0.
-        const target = this.context.createBuffer(
-          1,
-          source.length,
-          synth.sampleRate
-        );
-        const channel = target.getChannelData(0);
+        const target = new Float32Array(source.length);
         for (let i = 0; i < source.length; i++) {
           // Buffer is (0, 1023) we need to map it to (-1, 1)
-          channel[i] = (source[i] - 512) / 512;
+          target[i] = (source[i] - 512) / 512;
         }
         this.soundExpression!.writeData(target);
       }
@@ -201,6 +207,7 @@ export class BoardAudio {
     try {
       micStream = await navigator.mediaDevices.getUserMedia({
         video: false,
+        // It seems Firefox ignores the rate set here
         audio: true,
       });
     } catch (e) {
@@ -271,26 +278,41 @@ class BufferedAudio {
   constructor(
     private context: AudioContext,
     private destination: AudioNode,
+    private resampler: SRC,
     private callback: () => void
-  ) {}
+  ) {
+    this.resampler.outputSampleRate = this.context.sampleRate;
+  }
 
   init(sampleRate: number) {
     // This is called for each new audio source so don't reset nextStartTime
     // or we start to overlap audio
-    this.sampleRate = sampleRate;
-  }
-
-  createBuffer(length: number) {
-    // Use createBuffer instead of new AudioBuffer to support Safari 14.0.
-    return this.context.createBuffer(1, length, this.sampleRate);
+    this.setSampleRate(sampleRate);
   }
 
   setSampleRate(sampleRate: number) {
     this.sampleRate = sampleRate;
+    this.resampler.inputSampleRate = sampleRate;
   }
 
-  writeData(buffer: AudioBuffer) {
-    // Use createBufferSource instead of new AudioBufferSourceNode to support Safari 14.0.
+  writeData(data: Float32Array) {
+    let sampleRate = this.sampleRate;
+    // In practice the supported range is less than the 8k..96k required by the spec
+    if (sampleRate < 8_000 || sampleRate > 96_000) {
+      // We need to resample
+      //sampleRate = this.resampler.outputSampleRate;
+      //data = this.resampler.full(data);
+    }
+    console.log(
+      "Using actual rate",
+      sampleRate,
+      "for requested rate",
+      this.sampleRate
+    );
+
+    // Use createXXX instead to support Safari 14.0.
+    const buffer = this.context.createBuffer(1, data.length, sampleRate);
+    buffer.copyToChannel(data, 0);
     const source = this.context.createBufferSource();
     source.buffer = buffer;
     source.onended = this.callCallback;

Diff for: src/board/conversions.ts

@@ -98,12 +98,11 @@ export function convertAccelerometerNumberToString(value: number): string {
 export const convertAudioBuffer = (
   heap: Uint8Array,
   source: number,
-  target: AudioBuffer
+  target: Float32Array
 ) => {
-  const channel = target.getChannelData(0);
-  for (let i = 0; i < channel.length; ++i) {
+  for (let i = 0; i < target.length; ++i) {
     // Convert from uint8 to -1..+1 float.
-    channel[i] = (heap[source + i] / 255) * 2 - 1;
+    target[i] = (heap[source + i] / 255) * 2 - 1;
   }
   return target;
 };

Diff for: src/board/index.ts

@@ -1,3 +1,4 @@
+import { create as createResampler } from "@alexanderolsen/libsamplerate-js";
 import svgText from "../microbit-drawing.svg";
 import { Accelerometer } from "./accelerometer";
 import { BoardAudio } from "./audio";
@@ -247,10 +248,20 @@ export class Board {
       noInitialRun: true,
       instantiateWasm,
     });
+
+    // We update the sample rates before use.
+    const defaultResampler = await createResampler(1, 48000, 48000);
+    const speechResampler = await createResampler(1, 48000, 48000);
+    // Probably this one is never used so would be nice to avoid
+    const soundExpressionResampler = await createResampler(1, 48000, 48000);
+
     const module = new ModuleWrapper(wrapped);
     this.audio.initializeCallbacks({
       defaultAudioCallback: wrapped._microbit_hal_audio_raw_ready_callback,
+      defaultResampler,
       speechAudioCallback: wrapped._microbit_hal_audio_speech_ready_callback,
+      speechResampler,
+      soundExpressionResampler,
     });
     this.accelerometer.initializeCallbacks(
       wrapped._microbit_hal_gesture_callback

Diff for: src/jshal.js

@@ -230,8 +230,7 @@ mergeInto(LibraryManager.library, {
       Module.conversions.convertAudioBuffer(
         Module.HEAPU8,
         buf,
-        // @ts-expect-error
-        Module.board.audio.default.createBuffer(num_samples)
+        new Float32Array(num_samples)
       )
     );
   },
@@ -246,21 +245,10 @@ mergeInto(LibraryManager.library, {
     /** @type {number} */ num_samples
   ) {
     /** @type {AudioBuffer | undefined} */ let webAudioBuffer;
-    try {
-      // @ts-expect-error
-      webAudioBuffer = Module.board.audio.speech.createBuffer(num_samples);
-    } catch (e) {
-      // Swallow error on older Safari to keep the sim in a good state.
-      // @ts-expect-error
-      if (e.name === "NotSupportedError") {
-        return;
-      } else {
-        throw e;
-      }
-    }
+    const jsBuf = new Float32Array(num_samples);
     // @ts-expect-error
     Module.board.audio.speech.writeData(
-      Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, webAudioBuffer)
+      Module.conversions.convertAudioBuffer(Module.HEAPU8, buf, jsBuf)
     );
   },