From a29358af326fc792c9361ec7ce64e747dd9b35e7 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Sun, 7 Feb 2021 09:47:02 -0800 Subject: [PATCH] Support grabbing the pointer with the Pointer Lock API This change adds the following: a) A new button on the UI to enter full pointer lock mode, which invokes the Pointer Lock API[1] on the canvas, which hides the cursor and makes mouse events provide relative motion from the previous event (through `movementX` and `movementY`). These can be added to the previously-known mouse position to convert it back to an absolute position. b) Adds support for the VMware Cursor Position pseudo-encoding[2], which servers can use when they make cursor position changes themselves. This is done by some APIs like SDL, when they detect that the client does not support relative mouse movement[3] and then "warp"[4] the cursor to the center of the window, to calculate the relative mouse motion themselves. c) When the canvas is in pointer lock mode and the cursor is not being locally displayed, it updates the cursor position with the information that the server sends, since the actual position of the cursor does not matter locally anymore, since it's not visible. d) Adds some tests for the above. You can try this out end-to-end with TigerVNC with https://github.com/TigerVNC/tigervnc/pull/1198 applied! Fixes: #1493 under some circumstances (at least all SDL games would now work). 1: https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API 2: https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst#vmware-cursor-position-pseudo-encoding 3: https://hg.libsdl.org/SDL/file/28e3b60e2131/src/events/SDL_mouse.c#l804 4: https://tronche.com/gui/x/xlib/input/XWarpPointer.html --- app/images/pointer.svg | 78 ++++++++++++++++++++++++++++++++++++++++++ app/ui.js | 45 ++++++++++++++++++++++++ core/encodings.js | 1 + core/rfb.js | 65 ++++++++++++++++++++++++++++++++++- tests/test.rfb.js | 77 +++++++++++++++++++++++++++++++++++++++++ vnc.html | 5 +++ 6 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 app/images/pointer.svg diff --git a/app/images/pointer.svg b/app/images/pointer.svg new file mode 100644 index 000000000..dd394008e --- /dev/null +++ b/app/images/pointer.svg @@ -0,0 +1,78 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/app/ui.js b/app/ui.js index c1f6776ed..610a9682e 100644 --- a/app/ui.js +++ b/app/ui.js @@ -232,6 +232,10 @@ const UI = { document.getElementById("noVNC_view_drag_button") .addEventListener('click', UI.toggleViewDrag); + document + .getElementById("noVNC_pointer_lock_button") + .addEventListener("click", UI.requestPointerLock); + document.getElementById("noVNC_control_bar_handle") .addEventListener('mousedown', UI.controlbarHandleMouseDown); document.getElementById("noVNC_control_bar_handle") @@ -451,6 +455,7 @@ const UI = { UI.updatePowerButton(); UI.keepControlbar(); } + UI.updatePointerLockButton(); // State change closes dialogs as they may not be relevant // anymore @@ -1055,6 +1060,7 @@ const UI = { UI.rfb.addEventListener("clipboard", UI.clipboardReceive); UI.rfb.addEventListener("bell", UI.bell); UI.rfb.addEventListener("desktopname", UI.updateDesktopName); + UI.rfb.addEventListener("pointerlock", UI.pointerLockChanged); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; UI.rfb.resizeSession = UI.getSetting('resize') === 'remote'; @@ -1361,6 +1367,33 @@ const UI = { /* ------^------- * /VIEW CLIPPING * ============== + * POINTER LOCK + * ------v------*/ + + updatePointerLockButton() { + // Only show the button if the pointer lock API is properly supported + if ( + UI.connected && + (document.pointerLockElement !== undefined || + document.mozPointerLockElement !== undefined) + ) { + document + .getElementById("noVNC_pointer_lock_button") + .classList.remove("noVNC_hidden"); + } else { + document + .getElementById("noVNC_pointer_lock_button") + .classList.add("noVNC_hidden"); + } + }, + + requestPointerLock() { + UI.rfb.requestPointerLock(); + }, + +/* ------^------- + * /POINTER LOCK + * ============== * VIEWDRAG * ------v------*/ @@ -1729,6 +1762,18 @@ const UI = { document.title = e.detail.name + " - " + PAGE_TITLE; }, + pointerLockChanged(e) { + if (e.detail.pointerlock) { + document + .getElementById("noVNC_pointer_lock_button") + .classList.add("noVNC_selected"); + } else { + document + .getElementById("noVNC_pointer_lock_button") + .classList.remove("noVNC_selected"); + } + }, + bell(e) { if (WebUtil.getConfigVar('bell', 'on') === 'on') { const promise = document.getElementById('noVNC_bell').play(); diff --git a/core/encodings.js b/core/encodings.js index 2041b6e02..2b8202d1e 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -30,6 +30,7 @@ export const encodings = { pseudoEncodingCompressLevel9: -247, pseudoEncodingCompressLevel0: -256, pseudoEncodingVMwareCursor: 0x574d5664, + pseudoEncodingVMwareCursorPosition: 0x574d5666, pseudoEncodingExtendedClipboard: 0xc0a1e5ce }; diff --git a/core/rfb.js b/core/rfb.js index 6afd7c659..9e3962894 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -184,6 +184,7 @@ export default class RFB extends EventTargetMixin { this._mousePos = {}; this._mouseButtonMask = 0; this._mouseLastMoveTime = 0; + this._pointerLock = false; this._viewportDragging = false; this._viewportDragPos = {}; this._viewportHasMoved = false; @@ -201,6 +202,7 @@ export default class RFB extends EventTargetMixin { focusCanvas: this._focusCanvas.bind(this), handleResize: this._handleResize.bind(this), handleMouse: this._handleMouse.bind(this), + handlePointerLockChange: this._handlePointerLockChange.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), handleRSAAESCredentialsRequired: this._handleRSAAESCredentialsRequired.bind(this), @@ -493,6 +495,14 @@ export default class RFB extends EventTargetMixin { this._canvas.blur(); } + requestPointerLock() { + if (this._canvas.requestPointerLock) { + this._canvas.requestPointerLock(); + } else if (this._canvas.mozRequestPointerLock) { + this._canvas.mozRequestPointerLock(); + } + } + clipboardPasteFrom(text) { if (this._rfbConnectionState !== 'connected' || this._viewOnly) { return; } @@ -589,6 +599,8 @@ export default class RFB extends EventTargetMixin { // preventDefault() on mousedown doesn't stop this event for some // reason so we have to explicitly block it this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse); + // This needs to be installed in document instead of the canvas. + document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); // Wheel events this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); @@ -613,6 +625,7 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse); this._canvas.removeEventListener('click', this._eventHandlers.handleMouse); this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse); + document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); this._resizeObserver.disconnect(); @@ -1025,8 +1038,26 @@ export default class RFB extends EventTargetMixin { return; } - let pos = clientToElement(ev.clientX, ev.clientY, + let pos; + if (this._pointerLock) { + pos = { + x: this._mousePos.x + ev.movementX, + y: this._mousePos.y + ev.movementY, + }; + if (pos.x < 0) { + pos.x = 0; + } else if (pos.x > this._fbWidth) { + pos.x = this._fbWidth; + } + if (pos.y < 0) { + pos.y = 0; + } else if (pos.y > this._fbHeight) { + pos.y = this._fbHeight; + } + } else { + pos = clientToElement(ev.clientX, ev.clientY, this._canvas); + } switch (ev.type) { case 'mousedown': @@ -1127,6 +1158,20 @@ export default class RFB extends EventTargetMixin { this._mouseLastMoveTime = Date.now(); } + _handlePointerLockChange() { + if ( + document.pointerLockElement === this._canvas || + document.mozPointerLockElement === this._canvas + ) { + this._pointerLock = true; + } else { + this._pointerLock = false; + } + this.dispatchEvent(new CustomEvent( + "pointerlock", + { detail: { pointerlock: this._pointerLock }, })); + } + _sendMouse(x, y, mask) { if (this._rfbConnectionState !== 'connected') { return; } if (this._viewOnly) { return; } // View only, skip mouse events @@ -2170,6 +2215,8 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingCursor); } + encs.push(encodings.pseudoEncodingVMwareCursorPosition); + RFB.messages.clientEncodings(this._sock, encs); } @@ -2576,6 +2623,9 @@ export default class RFB extends EventTargetMixin { case encodings.pseudoEncodingVMwareCursor: return this._handleVMwareCursor(); + case encodings.pseudoEncodingVMwareCursorPosition: + return this._handleVMwareCursorPosition(); + case encodings.pseudoEncodingCursor: return this._handleCursor(); @@ -2714,6 +2764,19 @@ export default class RFB extends EventTargetMixin { return true; } + _handleVMwareCursorPosition() { + const x = this._FBU.x; + const y = this._FBU.y; + + if (this._pointerLock) { + // Only attempt to match the server's pointer position if we are in + // pointer lock mode. + this._mousePos = { x: x, y: y }; + } + + return true; + } + _handleCursor() { const hotx = this._FBU.x; // hotspot-x const hoty = this._FBU.y; // hotspot-y diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 2da381846..af45bb73e 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2822,6 +2822,27 @@ describe('Remote Frame Buffer Protocol Client', function () { client._canvas.dispatchEvent(ev); } + function supportsSendMouseMovementEvent() { + // Some browsers (like Safari) support the movementX / + // movementY properties of MouseEvent, but do not allow creation + // of non-trusted events with those properties. + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': 100, + 'movementY': 100 }); + return ev.movementX === 100 && ev.movementY === 100; + } + + function sendMouseMovementEvent(dx, dy) { + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': dx, + 'movementY': dy }); + client._canvas.dispatchEvent(ev); + } + function sendMouseButtonEvent(x, y, down, button) { let pos = elementToClient(x, y); let ev; @@ -2935,6 +2956,62 @@ describe('Remote Frame Buffer Protocol Client', function () { 50, 70, 0x0); }); + it('should ignore remote cursor position updates', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ }); + sendMouseMoveEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); + }); + + it('should handle remote mouse position updates in pointer lock mode', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const spy = sinon.spy(); + client.addEventListener("pointerlock", spy); + let stub = sinon.stub(document, 'pointerLockElement'); + stub.get(function () { return client._canvas; }); + client._handlePointerLockChange(); + stub.restore(); + client._sock._websocket._receiveData(new Uint8Array([0x02, 0x02])); + expect(spy).to.have.been.calledOnce; + expect(spy.args[0][0].detail.pointerlock).to.be.true; + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ x: 50, y: 50 }); + sendMouseMovementEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 60, 60, 0x0); + }); + describe('Event Aggregation', function () { it('should send a single pointer event on mouse movement', function () { sendMouseMoveEvent(50, 70); diff --git a/vnc.html b/vnc.html index 24a118dbd..b2f9f59a9 100644 --- a/vnc.html +++ b/vnc.html @@ -75,6 +75,11 @@

no
VNC

id="noVNC_view_drag_button" class="noVNC_button noVNC_hidden" title="Move/Drag Viewport"> + + +