Skip to content

Commit

Permalink
Support grabbing the pointer with the Pointer Lock API
Browse files Browse the repository at this point in the history 
This change adds the following:

a) A new button on the UI to enter full pointer lock mode, which invokes
   the Pointer Lock API[1] on the canvas, which hides the cursor and
   makes mouse events provide relative motion from the previous event
   (through `movementX` and `movementY`). These can be added to the
   previously-known mouse position to convert it back to an absolute
   position.
b) Adds support for the VMware Cursor Position pseudo-encoding[2], which
   servers can use when they make cursor position changes themselves.
   This is done by some APIs like SDL, when they detect that the client
   does not support relative mouse movement[3] and then "warp"[4] the
   cursor to the center of the window, to calculate the relative mouse
   motion themselves.
c) When the canvas is in pointer lock mode and the cursor is not being
   locally displayed, it updates the cursor position with the
   information that the server sends, since the actual position of the
   cursor does not matter locally anymore, since it's not visible.
d) Adds some tests for the above.

You can try this out end-to-end with TigerVNC with
TigerVNC/tigervnc#1198 applied!

Fixes: novnc#1493 under some circumstances (at least all SDL games would now
work).

1: https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API
2: https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst#vmware-cursor-position-pseudo-encoding
3: https://hg.libsdl.org/SDL/file/28e3b60e2131/src/events/SDL_mouse.c#l804
4: https://tronche.com/gui/x/xlib/input/XWarpPointer.html
  • Loading branch information
@lhchavez @aseering
lhchavez authored and aseering committed Feb 20, 2023
1 parent 90455ee commit a29358a
Show file tree
Hide file tree
Showing 6 changed files with 270 additions and 1 deletion.
78 changes: 78 additions & 0 deletions app/images/pointer.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
45 changes: 45 additions & 0 deletions app/ui.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,10 @@ const UI = {
document.getElementById("noVNC_view_drag_button")
.addEventListener('click', UI.toggleViewDrag);

document
.getElementById("noVNC_pointer_lock_button")
.addEventListener("click", UI.requestPointerLock);

document.getElementById("noVNC_control_bar_handle")
.addEventListener('mousedown', UI.controlbarHandleMouseDown);
document.getElementById("noVNC_control_bar_handle")
Expand Down Expand Up @@ -451,6 +455,7 @@ const UI = {
UI.updatePowerButton();
UI.keepControlbar();
}
UI.updatePointerLockButton();

// State change closes dialogs as they may not be relevant
// anymore
Expand Down Expand Up @@ -1055,6 +1060,7 @@ const UI = {
UI.rfb.addEventListener("clipboard", UI.clipboardReceive);
UI.rfb.addEventListener("bell", UI.bell);
UI.rfb.addEventListener("desktopname", UI.updateDesktopName);
UI.rfb.addEventListener("pointerlock", UI.pointerLockChanged);
UI.rfb.clipViewport = UI.getSetting('view_clip');
UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale';
UI.rfb.resizeSession = UI.getSetting('resize') === 'remote';
Expand Down Expand Up @@ -1361,6 +1367,33 @@ const UI = {
/* ------^-------
* /VIEW CLIPPING
* ==============
* POINTER LOCK
* ------v------*/

updatePointerLockButton() {
// Only show the button if the pointer lock API is properly supported
if (
UI.connected &&
(document.pointerLockElement !== undefined ||
document.mozPointerLockElement !== undefined)
) {
document
.getElementById("noVNC_pointer_lock_button")
.classList.remove("noVNC_hidden");
} else {
document
.getElementById("noVNC_pointer_lock_button")
.classList.add("noVNC_hidden");
}
},

requestPointerLock() {
UI.rfb.requestPointerLock();
},

/* ------^-------
* /POINTER LOCK
* ==============
* VIEWDRAG
* ------v------*/

Expand Down Expand Up @@ -1729,6 +1762,18 @@ const UI = {
document.title = e.detail.name + " - " + PAGE_TITLE;
},

pointerLockChanged(e) {
if (e.detail.pointerlock) {
document
.getElementById("noVNC_pointer_lock_button")
.classList.add("noVNC_selected");
} else {
document
.getElementById("noVNC_pointer_lock_button")
.classList.remove("noVNC_selected");
}
},

bell(e) {
if (WebUtil.getConfigVar('bell', 'on') === 'on') {
const promise = document.getElementById('noVNC_bell').play();
Expand Down
1 change: 1 addition & 0 deletions core/encodings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export const encodings = {
pseudoEncodingCompressLevel9: -247,
pseudoEncodingCompressLevel0: -256,
pseudoEncodingVMwareCursor: 0x574d5664,
pseudoEncodingVMwareCursorPosition: 0x574d5666,
pseudoEncodingExtendedClipboard: 0xc0a1e5ce
};

Expand Down
65 changes: 64 additions & 1 deletion core/rfb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ export default class RFB extends EventTargetMixin {
this._mousePos = {};
this._mouseButtonMask = 0;
this._mouseLastMoveTime = 0;
this._pointerLock = false;
this._viewportDragging = false;
this._viewportDragPos = {};
this._viewportHasMoved = false;
Expand All @@ -201,6 +202,7 @@ export default class RFB extends EventTargetMixin {
focusCanvas: this._focusCanvas.bind(this),
handleResize: this._handleResize.bind(this),
handleMouse: this._handleMouse.bind(this),
handlePointerLockChange: this._handlePointerLockChange.bind(this),
handleWheel: this._handleWheel.bind(this),
handleGesture: this._handleGesture.bind(this),
handleRSAAESCredentialsRequired: this._handleRSAAESCredentialsRequired.bind(this),
Expand Down Expand Up @@ -493,6 +495,14 @@ export default class RFB extends EventTargetMixin {
this._canvas.blur();
}

requestPointerLock() {
if (this._canvas.requestPointerLock) {
this._canvas.requestPointerLock();
} else if (this._canvas.mozRequestPointerLock) {
this._canvas.mozRequestPointerLock();
}
}

clipboardPasteFrom(text) {
if (this._rfbConnectionState !== 'connected' || this._viewOnly) { return; }

Expand Down Expand Up @@ -589,6 +599,8 @@ export default class RFB extends EventTargetMixin {
// preventDefault() on mousedown doesn't stop this event for some
// reason so we have to explicitly block it
this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse);
// This needs to be installed in document instead of the canvas.
document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange);

// Wheel events
this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel);
Expand All @@ -613,6 +625,7 @@ export default class RFB extends EventTargetMixin {
this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse);
this._canvas.removeEventListener('click', this._eventHandlers.handleMouse);
this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse);
document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange);
this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas);
this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas);
this._resizeObserver.disconnect();
Expand Down Expand Up @@ -1025,8 +1038,26 @@ export default class RFB extends EventTargetMixin {
return;
}

let pos = clientToElement(ev.clientX, ev.clientY,
let pos;
if (this._pointerLock) {
pos = {
x: this._mousePos.x + ev.movementX,
y: this._mousePos.y + ev.movementY,
};
if (pos.x < 0) {
pos.x = 0;
} else if (pos.x > this._fbWidth) {
pos.x = this._fbWidth;
}
if (pos.y < 0) {
pos.y = 0;
} else if (pos.y > this._fbHeight) {
pos.y = this._fbHeight;
}
} else {
pos = clientToElement(ev.clientX, ev.clientY,
this._canvas);
}

switch (ev.type) {
case 'mousedown':
Expand Down Expand Up @@ -1127,6 +1158,20 @@ export default class RFB extends EventTargetMixin {
this._mouseLastMoveTime = Date.now();
}

_handlePointerLockChange() {
if (
document.pointerLockElement === this._canvas ||
document.mozPointerLockElement === this._canvas
) {
this._pointerLock = true;
} else {
this._pointerLock = false;
}
this.dispatchEvent(new CustomEvent(
"pointerlock",
{ detail: { pointerlock: this._pointerLock }, }));
}

_sendMouse(x, y, mask) {
if (this._rfbConnectionState !== 'connected') { return; }
if (this._viewOnly) { return; } // View only, skip mouse events
Expand Down Expand Up @@ -2170,6 +2215,8 @@ export default class RFB extends EventTargetMixin {
encs.push(encodings.pseudoEncodingCursor);
}

encs.push(encodings.pseudoEncodingVMwareCursorPosition);

RFB.messages.clientEncodings(this._sock, encs);
}

Expand Down Expand Up @@ -2576,6 +2623,9 @@ export default class RFB extends EventTargetMixin {
case encodings.pseudoEncodingVMwareCursor:
return this._handleVMwareCursor();

case encodings.pseudoEncodingVMwareCursorPosition:
return this._handleVMwareCursorPosition();

case encodings.pseudoEncodingCursor:
return this._handleCursor();

Expand Down Expand Up @@ -2714,6 +2764,19 @@ export default class RFB extends EventTargetMixin {
return true;
}

_handleVMwareCursorPosition() {
const x = this._FBU.x;
const y = this._FBU.y;

if (this._pointerLock) {
// Only attempt to match the server's pointer position if we are in
// pointer lock mode.
this._mousePos = { x: x, y: y };
}

return true;
}

_handleCursor() {
const hotx = this._FBU.x; // hotspot-x
const hoty = this._FBU.y; // hotspot-y
Expand Down
77 changes: 77 additions & 0 deletions tests/test.rfb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2822,6 +2822,27 @@ describe('Remote Frame Buffer Protocol Client', function () {
client._canvas.dispatchEvent(ev);
}

function supportsSendMouseMovementEvent() {
// Some browsers (like Safari) support the movementX /
// movementY properties of MouseEvent, but do not allow creation
// of non-trusted events with those properties.
let ev;

ev = new MouseEvent('mousemove',
{ 'movementX': 100,
'movementY': 100 });
return ev.movementX === 100 && ev.movementY === 100;
}

function sendMouseMovementEvent(dx, dy) {
let ev;

ev = new MouseEvent('mousemove',
{ 'movementX': dx,
'movementY': dy });
client._canvas.dispatchEvent(ev);
}

function sendMouseButtonEvent(x, y, down, button) {
let pos = elementToClient(x, y);
let ev;
Expand Down Expand Up @@ -2935,6 +2956,62 @@ describe('Remote Frame Buffer Protocol Client', function () {
50, 70, 0x0);
});

it('should ignore remote cursor position updates', function () {
if (!supportsSendMouseMovementEvent()) {
this.skip();
return;
}
// Simple VMware Cursor Position FBU message with pointer coordinates
// (50, 50).
const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32,
0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ];
client._resize(100, 100);

const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition');
client._sock._websocket._receiveData(new Uint8Array(incoming));
expect(cursorSpy).to.have.been.calledOnceWith();
cursorSpy.restore();

expect(client._mousePos).to.deep.equal({ });
sendMouseMoveEvent(10, 10);
clock.tick(100);
expect(pointerEvent).to.have.been.calledOnceWith(client._sock,
10, 10, 0x0);
});

it('should handle remote mouse position updates in pointer lock mode', function () {
if (!supportsSendMouseMovementEvent()) {
this.skip();
return;
}
// Simple VMware Cursor Position FBU message with pointer coordinates
// (50, 50).
const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32,
0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ];
client._resize(100, 100);

const spy = sinon.spy();
client.addEventListener("pointerlock", spy);
let stub = sinon.stub(document, 'pointerLockElement');
stub.get(function () { return client._canvas; });
client._handlePointerLockChange();
stub.restore();
client._sock._websocket._receiveData(new Uint8Array([0x02, 0x02]));
expect(spy).to.have.been.calledOnce;
expect(spy.args[0][0].detail.pointerlock).to.be.true;

const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition');
client._sock._websocket._receiveData(new Uint8Array(incoming));
expect(cursorSpy).to.have.been.calledOnceWith();
cursorSpy.restore();

expect(client._mousePos).to.deep.equal({ x: 50, y: 50 });
sendMouseMovementEvent(10, 10);
clock.tick(100);
expect(pointerEvent).to.have.been.calledOnceWith(client._sock,
60, 60, 0x0);
});

describe('Event Aggregation', function () {
it('should send a single pointer event on mouse movement', function () {
sendMouseMoveEvent(50, 70);
Expand Down
Loading

0 comments on commit a29358a

Please sign in to comment.