Enable audio denoise using rnnoise-wasm

Author: arunetm

scripts/Gruntfile.js

@@ -5,7 +5,7 @@ module.exports = function(grunt) {
   const sdkOutput = 'dist/sdk/owt.js';
 
   var srcFiles = [
-    'src/sdk/base/**',
+    'src/sdk/base/*.js',
     'src/sdk/p2p/**',
     'src/sdk/conference/**'
   ];
@@ -175,6 +175,8 @@ window.L = L;\n\
     copy:{
       dist:{
         files:[
+          {expand: true,cwd:'src/sdk/base/',src:['*.wasm'],dest:'dist/samples/p2p/js',flatten:false},
+          {expand: true,cwd:'src/sdk/base/',src:['*.wasm'],dest:'dist/sdk/',flatten:false},
           {expand: true,cwd:'src/samples/p2p/',src:['**'],dest:'dist/samples/p2p/',flatten:false},
           {expand: true,cwd:'src/samples/conference/',src:['**'],dest:'dist/samples/conference/',flatten:false},
           {expand: true,cwd:'src/samples/conference/',src:['initcert.js'],dest:'dist/samples/conference/',flatten:false,mode:true},

src/samples/p2p/js/peercall.js

@@ -48,6 +48,10 @@ const getTargetId = function() {
   return $('#remote-uid').val();
 };
 
+function denoiseCheckboxChanged() {
+  document.getElementById("denoise-message").innerHTML=": Click 'Stop Camera' and 'share camera' if video sharing is already in progress."
+}
+
 $(document).ready(function() {
   $('#set-remote-uid').click(function() {
     p2p.allowedRemoteIds = [getTargetId()];
@@ -89,6 +93,8 @@ $(document).ready(function() {
     localStream = undefined;
   });
 
+  let denoiseCheckbox = document.getElementById('apply-denoise-checkbox');
+
   $('#target-video-publish').click(function() {
     $('#target-video-unpublish').prop('disabled', false);
     $('#target-video-publish').prop('disabled', true);
@@ -104,23 +110,44 @@ $(document).ready(function() {
       const videoConstraintsForCamera = new Owt.Base
         .VideoTrackConstraints(Owt.Base.VideoSourceInfo.CAMERA);
       let mediaStream;
-      Owt.Base.MediaStreamFactory.createMediaStream(new Owt.Base
-        .StreamConstraints(audioConstraintsForMic,
-          videoConstraintsForCamera)).then((stream) => {
-        mediaStream = stream;
-        localStream = new Owt.Base.LocalStream(mediaStream, new Owt
-          .Base.StreamSourceInfo('mic', 'camera'));
-        $('#local').children('video').get(0).srcObject = localStream
-          .mediaStream;
-        p2p.publish(getTargetId(), localStream).then(
-          (publication) => {
-            publicationForCamera = publication;
-          }, (error) => {
-            console.log('Failed to share video.');
-          });
-      }, (err) => {
-        console.error('Failed to create MediaStream, ' + err);
-      });
+      if(denoiseCheckbox.checked){
+        Owt.Base.MediaStreamFactory.createMediaStreamDenoised(new Owt.Base
+          .StreamConstraints(audioConstraintsForMic,
+            videoConstraintsForCamera)).then((stream) => {
+          mediaStream = stream;
+          localStream = new Owt.Base.LocalStream(mediaStream, new Owt
+            .Base.StreamSourceInfo('mic', 'camera'));
+          $('#local').children('video').get(0).srcObject = localStream
+            .mediaStream;
+          p2p.publish(getTargetId(), localStream).then(
+            (publication) => {
+              publicationForCamera = publication;
+            }, (error) => {
+              console.log('Failed to share video.');
+            });
+        }, (err) => {
+          console.error('Failed to create MediaStream, ' + err);
+        });
+      } 
+      else {
+        Owt.Base.MediaStreamFactory.createMediaStream(new Owt.Base
+          .StreamConstraints(audioConstraintsForMic,
+            videoConstraintsForCamera)).then((stream) => {
+          mediaStream = stream;
+          localStream = new Owt.Base.LocalStream(mediaStream, new Owt
+            .Base.StreamSourceInfo('mic', 'camera'));
+          $('#local').children('video').get(0).srcObject = localStream
+            .mediaStream;
+          p2p.publish(getTargetId(), localStream).then(
+            (publication) => {
+              publicationForCamera = publication;
+            }, (error) => {
+              console.log('Failed to share video.');
+            });
+        }, (err) => {
+          console.error('Failed to create MediaStream, ' + err);
+        });
+      }
     }
   });
 

src/samples/p2p/peercall.html

@@ -85,6 +85,12 @@ <h2>P2P Sample</h2>
       <input id="remote-uid" type="text" />
       <button id="set-remote-uid">Set Remote ID</button>
     </p>
+    <p>
+      <input id="apply-denoise-checkbox" type="checkbox" onclick="denoiseCheckboxChanged()">
+      <label for="apply-denoise-checkbox"> Enable noise reduction</label>
+      <mark><label id="denoise-message"> </label></mark>
+    </p>
+
     <p>
       <button id="target-video-publish">Share Camera</button>
       <button id="target-video-unpublish">Stop Camera Sharing</button>

src/sdk/base/denoise.js

@@ -0,0 +1,84 @@
+// Copyright (C) <2020> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/* global window, AudioContext, Float32Array */
+
+'use strict';
+// ///////////////////////////////////////////////////////////////////////////////
+// Handles the WebAssembly kernel for denoising raw audio files in F32Arrayformat
+// ///////////////////////////////////////////////////////////////////////////////
+
+import {Module, wasmMemory} from './rnn_denoise.js';
+
+// cwrap wasm API's used here make wasm calls from JS simpler.
+const wasmRnndenoiseRawmem = Module.cwrap('rnnDenoise_rawmem',
+    'number', ['number', 'number', 'number', 'number']);
+
+const sampleRate = 44100; // Audio Sample rate can be set and controlled here.
+const numChannels = 1; // Current channel support limited to 1. Stereo is ToDo.
+
+/**
+   * @function wasmDenoiseStream
+   * @desc Apply denoising into raw audio data in F32Array format using
+   * a WebAssembly port of RNNoise denoising algoritm.
+   * @return {Float32Array} fProcessdArr with denoised audio data
+   * @param {Float32Array} f32buffer
+   */
+export function wasmDenoiseStream(f32buffer) {
+  // Create and Initialize Wasm memory with input audio data.
+  const wasmMemPtr = Module._malloc(f32buffer.length * 4 );
+  const wasmMemArr = new Float32Array(wasmMemory.buffer,
+      wasmMemPtr, f32buffer.length);
+  wasmMemArr.set(f32buffer);
+
+  // Call Wasm denoising kernel
+  const wasmRetPtr = wasmRnndenoiseRawmem(wasmMemPtr,
+      sampleRate, numChannels, f32buffer.length);
+
+  // Create JS Array from Wasm memory with results
+  const fProcessedArr = new Float32Array(wasmMemory.buffer,
+      wasmRetPtr, f32buffer.length);
+
+  return fProcessedArr;
+}
+
+// ////////////////////////////////////////////////////////////////////
+// Creates a WebAudio Based filter for applying audio denoising.
+// ///////////////////////////////////////////////////////////////////
+
+// WebAuddio context
+window.AudioContext = window.AudioContext || window.webkitAudioContext;
+export const audioContext = new AudioContext(); // new AudioContext({sampleRate: 48000});
+
+
+// Audio buffer size:
+//     Accepts powers of 2 between 0 and 16384.
+//     Too low causes audio glitches due to buffer underruns
+//     Too high could increase latency.
+//     Set to 0 and WebAudio API will autopick a value.
+const bufferSize = 4096;
+
+export const audioDenoise = (function() {
+  const numberOfInputChannels = 1;
+  const numberOfOutputChannels = 1;
+  let recorder;
+
+  if (audioContext.createScriptProcessor) {
+    recorder = audioContext.createScriptProcessor(bufferSize,
+        numberOfInputChannels, numberOfOutputChannels);
+  } else {
+    recorder = audioContext.createJavaScriptNode(bufferSize,
+        numberOfInputChannels, numberOfOutputChannels);
+  }
+
+  recorder.onaudioprocess = function(e) {
+    const input = e.inputBuffer.getChannelData(0);
+
+    const wasmOutput = wasmDenoiseStream(input);
+
+    e.outputBuffer.copyToChannel(wasmOutput, 0, 0);
+  };
+  return recorder;
+})();
+

src/sdk/base/mediastream-factory.js

@@ -7,6 +7,7 @@
 'use strict';
 import * as utils from './utils.js';
 import * as MediaFormatModule from './mediaformat.js';
+import * as denoise from './denoise.js';
 
 /**
  * @class AudioTrackConstraints
@@ -226,4 +227,42 @@ export class MediaStreamFactory {
       return navigator.mediaDevices.getUserMedia(mediaConstraints);
     }
   }
+  /**
+   * @function createMediaStreamDenoised
+   * @static
+   * @desc Create a MediaStream with given constraints. Applies Rnnoise based
+   * noise cancellation on all audio streams. If you want to create a
+   * MediaStream for screen cast, please make sure both audio and video's source
+   * are "screen-cast".
+   * @memberof Owt.Base.MediaStreamFactory
+   * @return {Promise<MediaStream, Error>} Return a promise that is resolved
+   * when stream is successfully created, or rejected if one of the following
+   * error happened:
+   * - One or more parameters cannot be satisfied.
+   * - Specified device is busy.
+   * - Cannot obtain necessary permission or operation is canceled by user.
+   * - Video source is screen cast, while audio source is not.
+   * - Audio source is screen cast, while video source is disabled.
+   * @param {Owt.Base.StreamConstraints} constraints
+   */
+  static createMediaStreamDenoised(constraints) {
+    return this.createMediaStream(constraints).then(
+      (stream) => {
+        const audioTracks = stream.getAudioTracks();
+        const videoTracks = stream.getVideoTracks();
+        const peer = denoise.audioContext.createMediaStreamDestination();
+
+        audioTracks.forEach(function(track) {
+          const microphone =
+              denoise.audioContext.createMediaStreamSource(stream);
+          microphone.connect(denoise.audioDenoise);
+          denoise.audioDenoise.connect(peer);
+        });
+        videoTracks.forEach(function(track) {
+          peer.stream.addTrack(track);
+        });
+        return peer.stream;
+      }
+    );
+  }
 }