Skip to content

Remote cursor support (tested against UltraVNC and TightVNC so far) #1964

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jkanefendt wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Remote cursor support (tested against UltraVNC and TightVNC so far) #1964

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bce2a57
Remote cursor support (tested against UltraVNC and TightVNC so far)
jkanefendt May 28, 2025
e34a1c2
Remove opt-in for remote cursors. Remove redundant two's complement e…
jkanefendt Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions app/ui.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ const UI = {
UI.initSetting('shared', true);
UI.initSetting('bell', 'on');
UI.initSetting('view_only', false);
UI.initSetting('show_remote_cursor', true);
UI.initSetting('show_dot', false);
UI.initSetting('path', 'websockify');
UI.initSetting('repeaterID', '');
Expand Down Expand Up @@ -369,6 +370,8 @@ const UI = {
UI.addSettingChangeHandler('shared');
UI.addSettingChangeHandler('view_only');
UI.addSettingChangeHandler('view_only', UI.updateViewOnly);
UI.addSettingChangeHandler('show_remote_cursor');
UI.addSettingChangeHandler('show_remote_cursor', UI.updateShowRemoteCursor);
UI.addSettingChangeHandler('show_dot');
UI.addSettingChangeHandler('show_dot', UI.updateShowDotCursor);
UI.addSettingChangeHandler('host');
Expand Down Expand Up @@ -441,6 +444,7 @@ const UI = {
UI.disableSetting('port');
UI.disableSetting('path');
UI.disableSetting('repeaterID');
UI.disableSetting('show_remote_cursor');

// Hide the controlbar after 2 seconds
UI.closeControlbarTimeout = setTimeout(UI.closeControlbar, 2000);
Expand All @@ -451,6 +455,7 @@ const UI = {
UI.enableSetting('port');
UI.enableSetting('path');
UI.enableSetting('repeaterID');
UI.enableSetting('show_remote_cursor');
UI.updatePowerButton();
UI.keepControlbar();
}
Expand Down Expand Up @@ -887,6 +892,7 @@ const UI = {
UI.updateSetting('compression');
UI.updateSetting('shared');
UI.updateSetting('view_only');
UI.updateSetting('show_remote_cursor');
UI.updateSetting('path');
UI.updateSetting('repeaterID');
UI.updateSetting('logging');
Expand Down Expand Up @@ -1100,6 +1106,7 @@ const UI = {
UI.rfb.resizeSession = UI.getSetting('resize') === 'remote';
UI.rfb.qualityLevel = parseInt(UI.getSetting('quality'));
UI.rfb.compressionLevel = parseInt(UI.getSetting('compression'));
UI.rfb.showRemoteCursor = UI.getSetting('show_remote_cursor');
UI.rfb.showDotCursor = UI.getSetting('show_dot');

UI.updateViewOnly(); // requires UI.rfb
Expand Down Expand Up @@ -1754,6 +1761,11 @@ const UI = {
}
},

updateShowRemoteCursor () {
if (!UI.rfb) return;
UI.rfb.showRemoteCursor = UI.getSetting('show_remote_cursor');
},

updateShowDotCursor() {
if (!UI.rfb) return;
UI.rfb.showDotCursor = UI.getSetting('show_dot');
Expand Down
5 changes: 4 additions & 1 deletion core/encodings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ export const encodings = {
pseudoEncodingCompressLevel9: -247,
pseudoEncodingCompressLevel0: -256,
pseudoEncodingVMwareCursor: 0x574d5664,
pseudoEncodingExtendedClipboard: 0xc0a1e5ce
pseudoEncodingExtendedClipboard: 0xc0a1e5ce,
pseudoEncodingRichCursor: 0xffffff11,
Comment thread
CendioOssman marked this conversation as resolved.
Outdated
pseudoEncodingPointerPos: 0xffffff18,
pseudoEncodingTightPointerPos: -232
Comment thread
CendioOssman marked this conversation as resolved.
Outdated
};

export function encodingName(num) {
Expand Down
27 changes: 26 additions & 1 deletion core/rfb.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ export default class RFB extends EventTargetMixin {

get viewOnly() { return this._viewOnly; }
set viewOnly(viewOnly) {
this._viewOnly = viewOnly;
this._viewOnly = this._cursor.viewOnly = viewOnly;

if (this._rfbConnectionState === "connecting" ||
this._rfbConnectionState === "connected") {
Expand Down Expand Up @@ -413,6 +413,11 @@ export default class RFB extends EventTargetMixin {
}
}

get showRemoteCursor() { return this._showRemoteCursor; }
set showRemoteCursor(show) {
this._showRemoteCursor = show;
}

// ===== PUBLIC METHODS =====

disconnect() {
Expand Down Expand Up @@ -2264,6 +2269,12 @@ export default class RFB extends EventTargetMixin {
if (this._fbDepth == 24) {
encs.push(encodings.pseudoEncodingVMwareCursor);
encs.push(encodings.pseudoEncodingCursor);
encs.push(encodings.pseudoEncodingRichCursor);
}

if (this._showRemoteCursor) {
encs.push(encodings.pseudoEncodingPointerPos);
encs.push(encodings.pseudoEncodingTightPointerPos);
}

RFB.messages.clientEncodings(this._sock, encs);
Expand Down Expand Up @@ -2673,6 +2684,7 @@ export default class RFB extends EventTargetMixin {
return this._handleVMwareCursor();

case encodings.pseudoEncodingCursor:
case encodings.pseudoEncodingRichCursor:
return this._handleCursor();

case encodings.pseudoEncodingQEMUExtendedKeyEvent:
Expand All @@ -2696,6 +2708,10 @@ export default class RFB extends EventTargetMixin {
case encodings.pseudoEncodingQEMULedEvent:
return this._handleLedEvent();

case encodings.pseudoEncodingPointerPos:
case encodings.pseudoEncodingTightPointerPos:
return this._handlePointerPos();

default:
return this._handleDataRect();
}
Expand Down Expand Up @@ -2888,6 +2904,15 @@ export default class RFB extends EventTargetMixin {
return true;
}

_handlePointerPos() {
const x = this._FBU.x;
const y = this._FBU.y;

this._cursor.moveRemote(x, y, this._display.scale);
Copy link
Copy Markdown
Member

@CendioOssman CendioOssman May 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure that this is what this extension means? Not that the client is expected to really move the actual position of the cursor, e.g. as used in games?

Copy link
Copy Markdown
Author

@jkanefendt jkanefendt Jun 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's up to the client of how to deal with remote pointer events. The suggested implementation is drawing the remote cursor either if it's in view-only mode or if the local cursor is outside the canvas, otherwise such events are ignored.

Copy link
Copy Markdown
Member

@CendioOssman CendioOssman Jun 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The server needs to know what to expect, though. So the behaviour needs to be specified.

The UltraVNC code you linked treats it as just a display thing, which matches your PR here.

I also had a look at the original TightVNC code. It mostly does the same thing, with the exception of the X11 client. In there, it actually does XWarpCursor().

But since that isn't consistent, the server can't rely on it. So I'm willing to treat that as an odd exception and we can go with this just being for rendering.

As for when this needs to be done, it should be whenever needed to give the user the correct feedback. So it should follow the same logic the server uses without this extension.

In TigerVNC (which is based on the original RealVNC code), this is whenever the local cursor position and server cursor position differ, and some time has passed (to deal with various latencies). The TigerVNC code is rather crude, but in practice the time should be 500 ms on average.

That logic should hopefully handle view-only implicitly, as there will never be a valid local cursor position in that case.


return true;
}

_handleExtendedDesktopSize() {
if (this._sock.rQwait("ExtendedDesktopSize", 4)) {
return false;
Expand Down
116 changes: 90 additions & 26 deletions core/util/cursor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,15 @@ export default class Cursor {

this._canvas = document.createElement('canvas');

if (useFallback) {
this._canvas.style.position = 'fixed';
this._canvas.style.zIndex = '65535';
this._canvas.style.pointerEvents = 'none';
// Safari on iOS can select the cursor image
// https://bugs.webkit.org/show_bug.cgi?id=249223
this._canvas.style.userSelect = 'none';
this._canvas.style.WebkitUserSelect = 'none';
// Can't use "display" because of Firefox bug #1445997
this._canvas.style.visibility = 'hidden';
}
this._canvas.style.position = 'fixed';
this._canvas.style.zIndex = '65535';
this._canvas.style.pointerEvents = 'none';
// Safari on iOS can select the cursor image
// https://bugs.webkit.org/show_bug.cgi?id=249223
this._canvas.style.userSelect = 'none';
this._canvas.style.WebkitUserSelect = 'none';
// Can't use "display" because of Firefox bug #1445997
this._canvas.style.visibility = 'hidden';

this._position = { x: 0, y: 0 };
this._hotSpot = { x: 0, y: 0 };
Expand All @@ -35,6 +33,20 @@ export default class Cursor {
'mousemove': this._handleMouseMove.bind(this),
'mouseup': this._handleMouseUp.bind(this),
};

this._mouseOver = false;
this._viewOnly = false;
}

get viewOnly() { return this._viewOnly; }
set viewOnly(viewOnly) {
if (viewOnly !== this._viewOnly) {
this._viewOnly = viewOnly;
this._resetNativeCursorStyle();
if (this._viewOnly) {
this._showCursor();
}
}
}

attach(target) {
Expand All @@ -44,12 +56,13 @@ export default class Cursor {

this._target = target;

if (useFallback) {
document.body.appendChild(this._canvas);
document.body.appendChild(this._canvas);

const options = { capture: true, passive: true };
this._target.addEventListener('mouseover', this._eventHandlers.mouseover, options);
this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options);

const options = { capture: true, passive: true };
this._target.addEventListener('mouseover', this._eventHandlers.mouseover, options);
this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options);
if (useFallback) {
this._target.addEventListener('mousemove', this._eventHandlers.mousemove, options);
this._target.addEventListener('mouseup', this._eventHandlers.mouseup, options);
}
Expand All @@ -62,16 +75,17 @@ export default class Cursor {
return;
}

const options = { capture: true, passive: true };
this._target.removeEventListener('mouseover', this._eventHandlers.mouseover, options);
this._target.removeEventListener('mouseleave', this._eventHandlers.mouseleave, options);

if (useFallback) {
const options = { capture: true, passive: true };
this._target.removeEventListener('mouseover', this._eventHandlers.mouseover, options);
this._target.removeEventListener('mouseleave', this._eventHandlers.mouseleave, options);
this._target.removeEventListener('mousemove', this._eventHandlers.mousemove, options);
this._target.removeEventListener('mouseup', this._eventHandlers.mouseup, options);
}

if (document.contains(this._canvas)) {
document.body.removeChild(this._canvas);
}
if (document.contains(this._canvas)) {
document.body.removeChild(this._canvas);
}

this._target = null;
Expand All @@ -97,16 +111,17 @@ export default class Cursor {
ctx.clearRect(0, 0, w, h);
ctx.putImageData(img, 0, 0);

if (useFallback) {
if (useFallback || this._viewOnly || !this._mouseOver) {
this._updatePosition();
} else {
}
if (!useFallback && !this._viewOnly) {
let url = this._canvas.toDataURL();
this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default';
}
}

clear() {
this._target.style.cursor = 'none';
this._resetNativeCursorStyle();
this._canvas.width = 0;
this._canvas.height = 0;
this._position.x = this._position.x + this._hotSpot.x;
Expand All @@ -115,6 +130,12 @@ export default class Cursor {
this._hotSpot.y = 0;
}

_resetNativeCursorStyle() {
if (this._target) {
this._target.style.cursor = this._viewOnly ? 'not-allowed' : 'none';
}
}

// Mouse events might be emulated, this allows
// moving the cursor in such cases
move(clientX, clientY) {
Expand All @@ -136,19 +157,58 @@ export default class Cursor {
this._updateVisibility(target);
}

moveRemote(remoteX, remoteY, scale) {
if (this._mouseOver && !this._viewOnly) {
return;
}

let targetBounds = this._target.getBoundingClientRect();
this._position.x = targetBounds.left + remoteX * scale - this._hotSpot.x;
this._position.y = targetBounds.top + remoteY * scale - this._hotSpot.y;

this._updatePosition();
}

_handleMouseOver(event) {
// This event could be because we're entering the target, or
// moving around amongst its sub elements. Let the move handler
// sort things out.
this._mouseOver = true;
this._handleMouseMove(event);
}

_handleMouseLeave(event) {
if (this._viewOnly) {
return;
}

let targetBounds = this._getVisibleBoundingRect(this._target);
this._mouseOver = event.clientX >= targetBounds.left && event.clientX < targetBounds.right &&
event.clientY >= targetBounds.top && event.clientY < targetBounds.bottom;
// Check if we should show the cursor on the element we are leaving to
this._updateVisibility(event.relatedTarget);
}

_getVisibleBoundingRect(element) {
let rect = element.getBoundingClientRect();
let bounds = { left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom };
if (element.parentElement) {
let parentBounds = element.parentElement.getBoundingClientRect();
bounds = {
left: Math.max(bounds.left, parentBounds.left),
top: Math.max(bounds.top, parentBounds.top),
right: Math.min(bounds.right, parentBounds.right),
bottom: Math.min(bounds.bottom, parentBounds.bottom)
};
}
return bounds;
}

_handleMouseMove(event) {
if (this._viewOnly) {
return;
}

this._updateVisibility(event.target);

this._position.x = event.clientX - this._hotSpot.x;
Expand All @@ -158,6 +218,10 @@ export default class Cursor {
}

_handleMouseUp(event) {
if (this._viewOnly) {
return;
}

// We might get this event because of a drag operation that
// moved outside of the target. Check what's under the cursor
// now and adjust visibility based on that.
Expand Down Expand Up @@ -230,7 +294,7 @@ export default class Cursor {
if (this._captureIsActive()) {
target = document.captureElement;
}
if (this._shouldShowCursor(target)) {
if (!this._mouseOver || (useFallback && this._shouldShowCursor(target))) {
this._showCursor();
} else {
this._hideCursor();
Expand Down
4 changes: 4 additions & 0 deletions docs/API.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ protocol stream.
if the remote session is smaller than its container, or handled
according to `clipViewport` if it is larger. Disabled by default.

`showRemoteCursor`
- Is a `boolean` indicating whether the remote cursor position should
be tracked. The server must support the respective pseudo encoding.

Copy link
Copy Markdown
Member

@CendioOssman CendioOssman May 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is truly just a rendering optimization, then we don't need a setting. Just have it enabled.

Copy link
Copy Markdown
Author

@jkanefendt jkanefendt Jun 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e34a1c2

`showDotCursor`
- Is a `boolean` indicating whether a dot cursor should be shown
instead of a zero-sized or fully-transparent cursor if the server
Expand Down
7 changes: 7 additions & 0 deletions vnc.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,13 @@ <h1 class="noVNC_logo" translate="no"><span>no</span><br>VNC</h1>
<input id="noVNC_setting_reconnect_delay" type="number">
</li>
<li><hr></li>
<li>
<label>
<input id="noVNC_setting_show_remote_cursor" type="checkbox"
class="toggle">
Show remote cursor
</label>
</li>
<li>
<label>
<input id="noVNC_setting_show_dot" type="checkbox"
Expand Down