From e9b1325a3c379f020b8da272ada4a06720c034e1 Mon Sep 17 00:00:00 2001 From: tinyzimmer <38474291+tinyzimmer@users.noreply.github.com> Date: Thu, 27 Aug 2020 17:55:30 +0300 Subject: [PATCH 1/4] first draft audio support --- core/audio.js | 43 ++++++++++++++++++++++++++ core/rfb.js | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 core/audio.js diff --git a/core/audio.js b/core/audio.js new file mode 100644 index 000000000..20b6ad9a1 --- /dev/null +++ b/core/audio.js @@ -0,0 +1,43 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2020 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + */ + +export default class AudioBuffer { + constructor(codec) { + // instantiate a media source and audio buffer/queue + this._mediaSource = new MediaSource(); + this._audioBuffer = null; + this._audioQ = []; + + // create a hidden audio element + this._audio = document.createElement('audio'); + this._audio.src = window.URL.createObjectURL(mediaSource); + + // when data is queued, start playing + this._mediaSource.addEventListener('sourceopen', function(e) { + this._audio.play(); + this._audioBuffer = mediaSource.addSourceBuffer(codec) + this._audioBuffer.addEventListener('update', function() { + if (this._audioQ.length > 0 && !this._audioBuffer.updating) { + this._audioBuffer.appendBuffer(this._audioQ.shift()) + } + }) + }, false) + } + + queueAudio(data) { + if (this._audioBuffer !== null) { + if (this._audioBuffer.updating || this._audioQ.length > 0) { + this._audioQ.push(data) + } else { + this._audioBuffer.appendBuffer(data) + } + } + } + + close() {} // intentionally left empty as no cleanup seems necessary +} \ No newline at end of file diff --git a/core/rfb.js b/core/rfb.js index f35d503f1..f16ef3153 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -499,6 +499,15 @@ export default class RFB extends EventTargetMixin { } } + enableAudio(sampleFormat, channels, frequency) { + RFB.messages.SetQEMUExtendedAudioFormat(this._sock, sampleFormat, channels, frequency) + RFB.messages.ToggleQEMUExtendedAudio(this._sock, true) + } + + disableAudio() { + RFB.messages.ToggleQEMUExtendedAudio(this._sock, false) + } + // ===== PRIVATE METHODS ===== _connect() { @@ -2039,6 +2048,25 @@ export default class RFB extends EventTargetMixin { return true; } + _handleQEMUExtAudioMsg() { + if (this._sock.rQwait("QEMU extended audio message", 3, 1)) { return false; } + + this._sock.rQshift8(); // for now there is only a single submessage type 1 + const operation = this._sock.rQshift16(); + + if (operation === 1) { // stream is starting + this._audioBuffer = new AudioBuffer('audio/webm; codecs="opus"') // TODO: This is obviously not the right value to use here + } else if (operation === 0) { // stream is stopping + this._audioBuffer.close() + } else { // stream data + const length = this._sock.rQshift32(); + const data = this._sock.rQshiftBytes(length); + this._audioBuffer.queueAudio(data) + } + + return true; + } + _handleXvpMsg() { if (this._sock.rQwait("XVP version and message", 3, 1)) { return false; } this._sock.rQskipBytes(1); // Padding @@ -2112,6 +2140,9 @@ export default class RFB extends EventTargetMixin { case 250: // XVP return this._handleXvpMsg(); + + case 255: // Qemu extended audio message + return this._handleQEMUExtAudioMsg(); default: this._fail("Unexpected server message (type " + msgType + ")"); @@ -2549,6 +2580,16 @@ export default class RFB extends EventTargetMixin { } } +// Audio sample formats +RFB.sampleFormats = { + U8: 0, + S8: 1, + U16: 2, + S16: 3, + U32: 4, + S32: 5 +} + // Class Methods RFB.messages = { keyEvent(sock, keysym, down) { @@ -2570,6 +2611,48 @@ RFB.messages = { sock.flush(); }, + ToggleQEMUExtendedAudio(sock, enabled) { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 255; // msg-type + buff[offset + 1] = 1; // sub msg-type + + buff[offset + 2] = 0; // operation + if (enabled) { + buff[offset + 3] = 0; + } else { + buff[offset + 3] = 1; + } + + sock._sQlen += 4; + sock.flush(); + }, + + SetQEMUExtendedAudioFormat(sock, sampleFormat, channels, frequency) { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 255; // msg type + buff[offset + 1] = 1; // sub msg-type + + buff[offset + 2] = 0; // operation + buff[offset + 3] = 2; + + buff[offset + 4] = sampleFormat; + buff[offset + 5] = channels; + + const freq = toUnsigned32bit(frequency); + + buff[offset + 6] = freq >> 24; + buff[offset + 7] = freq >> 16; + buff[offset + 8] = freq >> 8; + buff[offset + 9] = freq; + + sock._sQlen += 10; + sock.flush() + }, + QEMUExtendedKeyEvent(sock, keysym, down, keycode) { function getRFBkeycode(xtScanCode) { const upperByte = (keycode >> 8); From 5a4e8d216e83950b8f3f8242e3ccbffdfcecff95 Mon Sep 17 00:00:00 2001 From: tinyzimmer <38474291+tinyzimmer@users.noreply.github.com> Date: Thu, 27 Aug 2020 18:08:03 +0300 Subject: [PATCH 2/4] move audio.js into util package --- core/rfb.js | 1 + core/{ => util}/audio.js | 0 2 files changed, 1 insertion(+) rename core/{ => util}/audio.js (100%) diff --git a/core/rfb.js b/core/rfb.js index f16ef3153..6b1668e06 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -13,6 +13,7 @@ import { encodeUTF8, decodeUTF8 } from './util/strings.js'; import { dragThreshold } from './util/browser.js'; import { clientToElement } from './util/element.js'; import { setCapture } from './util/events.js'; +import AudioBuffer from './util/audio.js'; import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Inflator from "./inflator.js"; diff --git a/core/audio.js b/core/util/audio.js similarity index 100% rename from core/audio.js rename to core/util/audio.js From c2b6e8e13d29277a04b69feac6d91671453c164a Mon Sep 17 00:00:00 2001 From: tinyzimmer <38474291+tinyzimmer@users.noreply.github.com> Date: Thu, 27 Aug 2020 18:15:03 +0300 Subject: [PATCH 3/4] linting and syntax errors ; will squash these commits before an actual PR --- core/rfb.js | 18 +++++++++--------- core/util/audio.js | 28 ++++++++++++++++------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index 6b1668e06..c5f3cd7cb 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -501,12 +501,12 @@ export default class RFB extends EventTargetMixin { } enableAudio(sampleFormat, channels, frequency) { - RFB.messages.SetQEMUExtendedAudioFormat(this._sock, sampleFormat, channels, frequency) - RFB.messages.ToggleQEMUExtendedAudio(this._sock, true) + RFB.messages.SetQEMUExtendedAudioFormat(this._sock, sampleFormat, channels, frequency); + RFB.messages.ToggleQEMUExtendedAudio(this._sock, true); } disableAudio() { - RFB.messages.ToggleQEMUExtendedAudio(this._sock, false) + RFB.messages.ToggleQEMUExtendedAudio(this._sock, false); } // ===== PRIVATE METHODS ===== @@ -2056,13 +2056,13 @@ export default class RFB extends EventTargetMixin { const operation = this._sock.rQshift16(); if (operation === 1) { // stream is starting - this._audioBuffer = new AudioBuffer('audio/webm; codecs="opus"') // TODO: This is obviously not the right value to use here + this._audioBuffer = new AudioBuffer('audio/webm; codecs="opus"'); // TODO: This is obviously not the right value to use here } else if (operation === 0) { // stream is stopping - this._audioBuffer.close() + this._audioBuffer.close(); } else { // stream data const length = this._sock.rQshift32(); const data = this._sock.rQshiftBytes(length); - this._audioBuffer.queueAudio(data) + this._audioBuffer.queueAudio(data); } return true; @@ -2141,7 +2141,7 @@ export default class RFB extends EventTargetMixin { case 250: // XVP return this._handleXvpMsg(); - + case 255: // Qemu extended audio message return this._handleQEMUExtAudioMsg(); @@ -2589,7 +2589,7 @@ RFB.sampleFormats = { S16: 3, U32: 4, S32: 5 -} +}; // Class Methods RFB.messages = { @@ -2651,7 +2651,7 @@ RFB.messages = { buff[offset + 9] = freq; sock._sQlen += 10; - sock.flush() + sock.flush(); }, QEMUExtendedKeyEvent(sock, keysym, down, keycode) { diff --git a/core/util/audio.js b/core/util/audio.js index 20b6ad9a1..8ef651966 100644 --- a/core/util/audio.js +++ b/core/util/audio.js @@ -15,26 +15,30 @@ export default class AudioBuffer { // create a hidden audio element this._audio = document.createElement('audio'); - this._audio.src = window.URL.createObjectURL(mediaSource); + this._audio.src = window.URL.createObjectURL(this._mediaSource); // when data is queued, start playing - this._mediaSource.addEventListener('sourceopen', function(e) { - this._audio.play(); - this._audioBuffer = mediaSource.addSourceBuffer(codec) - this._audioBuffer.addEventListener('update', function() { - if (this._audioQ.length > 0 && !this._audioBuffer.updating) { - this._audioBuffer.appendBuffer(this._audioQ.shift()) - } - }) - }, false) + this._mediaSource.addEventListener('sourceopen', this._onSourceOpen, false); + } + + _onSourceOpen(e) { + this._audio.play(); + this._audioBuffer = this._mediaSource.addSourceBuffer(codec); + this._audioBuffer.addEventListener('update', this._onUpdateBuffer); + } + + _onUpdateBuffer() { + if (this._audioQ.length > 0 && !this._audioBuffer.updating) { + this._audioBuffer.appendBuffer(this._audioQ.shift()); + } } queueAudio(data) { if (this._audioBuffer !== null) { if (this._audioBuffer.updating || this._audioQ.length > 0) { - this._audioQ.push(data) + this._audioQ.push(data); } else { - this._audioBuffer.appendBuffer(data) + this._audioBuffer.appendBuffer(data); } } } From 1d1fc3811dad194abd840c9aeff718e541868c8a Mon Sep 17 00:00:00 2001 From: tinyzimmer <38474291+tinyzimmer@users.noreply.github.com> Date: Thu, 27 Aug 2020 18:18:09 +0300 Subject: [PATCH 4/4] set internal _codec attribute for AudioBuffer --- core/util/audio.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/util/audio.js b/core/util/audio.js index 8ef651966..85a8a7a5a 100644 --- a/core/util/audio.js +++ b/core/util/audio.js @@ -8,6 +8,7 @@ export default class AudioBuffer { constructor(codec) { + this._codec = codec // instantiate a media source and audio buffer/queue this._mediaSource = new MediaSource(); this._audioBuffer = null; @@ -23,7 +24,7 @@ export default class AudioBuffer { _onSourceOpen(e) { this._audio.play(); - this._audioBuffer = this._mediaSource.addSourceBuffer(codec); + this._audioBuffer = this._mediaSource.addSourceBuffer(this._codec); this._audioBuffer.addEventListener('update', this._onUpdateBuffer); }