Qemu audio support

app/ui.js

@@ -189,6 +189,7 @@ const UI = {
         UI.initSetting('repeaterID', '');
         UI.initSetting('reconnect', false);
         UI.initSetting('reconnect_delay', 5000);
+        UI.initSetting('enable_audio', true);
     },
     // Adds a link to the label elements on the corresponding input elements
     setupSettingLabels() {
@@ -333,6 +334,12 @@ const UI = {
             .addEventListener('click', UI.rejectServer);
         document.getElementById("noVNC_credentials_button")
             .addEventListener('click', UI.setCredentials);
+
+        document.addEventListener('click', function(event) {
+            if (UI.rfb !== undefined) {
+                UI.rfb.allow_audio();
+            }
+        });
     },
 
     addClipboardHandlers() {
@@ -379,6 +386,8 @@ const UI = {
         UI.addSettingChangeHandler('logging', UI.updateLogging);
         UI.addSettingChangeHandler('reconnect');
         UI.addSettingChangeHandler('reconnect_delay');
+        UI.addSettingChangeHandler('enable_audio');
+        UI.addSettingChangeHandler('enable_audio', UI.updateEnableAudio);
     },
 
     addFullscreenHandlers() {
@@ -892,6 +901,7 @@ const UI = {
         UI.updateSetting('logging');
         UI.updateSetting('reconnect');
         UI.updateSetting('reconnect_delay');
+        UI.updateSetting('enable_audio');
 
         document.getElementById('noVNC_settings')
             .classList.add("noVNC_open");
@@ -1103,6 +1113,8 @@ const UI = {
         UI.rfb.showDotCursor = UI.getSetting('show_dot');
 
         UI.updateViewOnly(); // requires UI.rfb
+        UI.updateEnableAudio(); // requires UI.rfb
+
     },
 
     disconnect() {
@@ -1795,6 +1807,11 @@ const UI = {
         selectbox.options.add(optn);
     },
 
+    updateEnableAudio() {
+        if (!UI.rfb) return;
+        UI.rfb.enable_audio(UI.getSetting('enable_audio'));
+    },
+
 /* ------^-------
  *    /MISC
  * ==============

core/encodings.js

@@ -24,6 +24,7 @@ export const encodings = {
     pseudoEncodingLastRect: -224,
     pseudoEncodingCursor: -239,
     pseudoEncodingQEMUExtendedKeyEvent: -258,
+    pseudoEncodingQEMUAudioEvent: -259,
     pseudoEncodingQEMULedEvent: -261,
     pseudoEncodingDesktopName: -307,
     pseudoEncodingExtendedDesktopSize: -308,

core/rfb.js

@@ -154,6 +154,14 @@ export default class RFB extends EventTargetMixin {
 
         this._qemuExtKeyEventSupported = false;
 
+        this._qemuAudioSupported = false;
+        this._page_had_user_interaction = false;
+        this._audio_enable = false;
+        this._audio_next_start = 0;
+        this._audio_sample_rate = 44100;
+        this._audio_channels = 2;
+        this._audio_context = null;
+
         this._extendedPointerEventSupported = false;
 
         this._clipboardText = null;
@@ -991,7 +999,6 @@ export default class RFB extends EventTargetMixin {
             Log.Warn("handleMessage called on an empty receive queue");
             return;
         }
-
         switch (this._rfbConnectionState) {
             case 'disconnected':
                 Log.Error("Got data while disconnected");
@@ -2256,6 +2263,7 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.pseudoEncodingDesktopSize);
         encs.push(encodings.pseudoEncodingLastRect);
         encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent);
+        encs.push(encodings.pseudoEncodingQEMUAudioEvent);
         encs.push(encodings.pseudoEncodingQEMULedEvent);
         encs.push(encodings.pseudoEncodingExtendedDesktopSize);
         encs.push(encodings.pseudoEncodingXvp);
@@ -2611,6 +2619,9 @@ export default class RFB extends EventTargetMixin {
             case 250:  // XVP
                 return this._handleXvpMsg();
 
+            case 255: // Qemu Server Message
+                return this._handleQemuAudioEvent();
+
             default:
                 this._fail("Unexpected server message (type " + msgType + ")");
                 Log.Debug("sock.rQpeekBytes(30): " + this._sock.rQpeekBytes(30));
@@ -2683,6 +2694,13 @@ export default class RFB extends EventTargetMixin {
                 this._qemuExtKeyEventSupported = true;
                 return true;
 
+            case encodings.pseudoEncodingQEMUAudioEvent:
+                if (!this._qemuAudioSupported) {
+                    RFB.messages.enableQemuAudioUpdates(this._sock, this._audio_channels, this._audio_sample_rate);
+                    this._qemuAudioSupported = true;
+                }
+                return true;
+
             case encodings.pseudoEncodingDesktopName:
                 return this._handleDesktopName();
 
@@ -2705,6 +2723,101 @@ export default class RFB extends EventTargetMixin {
         }
     }
 
+    _handleQemuAudioEvent() {
+        if (this._sock.rQwait("Qemu Audio Event", 3, 1)) {
+            return false;
+        }
+
+        const submsg = this._sock.rQshift8();
+        const operation = this._sock.rQshift16();
+
+        switch (operation) {
+            case 0: {
+                this._audio_context = null;
+                this._audio_next_start = 0;
+                return true;
+            }
+            case 1: {
+                this._audio_context = new AudioContext({
+                    latencyHint: "interactive",
+                    sampleRate: this._audio_sample_rate,
+                });
+                this._audio_next_start = 0;
+                return true;
+            }
+            case 2: break;
+            default: {
+                Log.Warn("The given qemu audio opertaion + opertaion + is not supported.");
+                return false;
+            }
+        }
+
+        if (this._sock.rQwait("Qemu Audio payload length", 4, 4)) {
+            return false;
+        }
+
+        const length = this._sock.rQshift32();
+
+        if (this._sock.rQwait("audio payload", length, 8)) {
+            return false;
+        }
+
+        if (length !== 0) {
+            let payload = this._sock.rQshiftBytes(length, false);
+
+            if (this._audio_context === null) {
+                return false;
+            }
+
+            let sample_bytes = 2*this._audio_channels;
+            let buffer = this._audio_context.createBuffer(this._audio_channels, length/sample_bytes, this._audio_sample_rate);
+
+            for (let ch = 0; ch < this._audio_channels; ch++) {
+                const channel = buffer.getChannelData(ch);
+                let channel_offset = ch*2;
+                for (let i = 0; i < buffer.length; i++) {
+                    let p = i*sample_bytes + channel_offset;
+                    let value = payload[p] + payload[p+1]*256;
+                    channel[i] = (value / 32768.0) - 1.0;
+                }
+            }
+
+            if (this._page_had_user_interaction && this._audio_enable) {
+                let ctime = this._audio_context.currentTime;
+                if (ctime > this._audio_next_start) {
+                    this._audio_next_start = ctime;
+                }
+                let start_time = this._audio_next_start;
+
+                this._audio_next_start += buffer.duration;
+
+                let source = this._audio_context.createBufferSource();
+                source.buffer = buffer;
+                source.connect(this._audio_context.destination);
+                source.start(start_time);
+            }
+        }
+
+        return true;
+    }
+
+    enable_audio(value) {
+        if (this._audio_enable !== value) {
+            this._audio_enable = value;
+            if (this._qemuAudioSupported) {
+                if (this._audio_enable) {
+                    RFB.messages.enableQemuAudioUpdates(this._sock, this._audio_channels, this._audio_sample_rate);
+                } else {
+                    RFB.messages.disableQemuAudioUpdates(this._sock);
+                }
+            }
+        }
+    }
+
+    allow_audio() {
+        this._page_had_user_interaction = true;
+    }
+
     _handleVMwareCursor() {
         const hotx = this._FBU.x;  // hotspot-x
         const hoty = this._FBU.y;  // hotspot-y
@@ -3314,6 +3427,30 @@ RFB.messages = {
         sock.flush();
     },
 
+    disableQemuAudioUpdates(sock, channels, sample_rate) {
+        sock.sQpush8(255); // msg-type
+        sock.sQpush8(1); // submessage-type
+        sock.sQpush16(1); // disable audio
+
+        sock.flush();
+    },
+
+    enableQemuAudioUpdates(sock, channels, sample_rate) {
+
+        sock.sQpush8(255); // msg-type
+        sock.sQpush8(1); // submessage-type
+        sock.sQpush16(2); // set sample format
+        sock.sQpush8(2); // format U16
+        sock.sQpush8(channels);
+        sock.sQpush32(sample_rate); // audio frequency
+
+        sock.sQpush8(255); // msg-type
+        sock.sQpush8(1); // submessage-type
+        sock.sQpush16(0); // enable audio
+
+        sock.flush();
+    },
+
     pixelFormat(sock, depth, trueColor) {
         let bpp;
 

vnc.html

@@ -219,6 +219,13 @@ <h1 class="noVNC_logo" translate="no"><span>no</span><br>VNC</h1>
                             View only
                         </label>
                     </li>
+                    <li>
+                        <label>
+                            <input id="noVNC_setting_enable_audio" type="checkbox"
+                                   class="toggle">
+                            Enable Audio
+                        </label>
+                    </li>
                     <li><hr></li>
                     <li>
                         <label>