From ee30b6a1ccaab6f44a99acf110c1d45e85ba089b Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:06:45 +0100 Subject: [PATCH 01/47] add settings for hand tracking --- alvr/session/src/settings.rs | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index cc56e1a4e1..831a1c7b4a 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -659,6 +659,9 @@ pub struct ControllersConfig { pub button_mapping_config: AutomaticButtonMappingConfig, + #[schema(strings(help = "Turning this off will disable hand-tracking."))] + pub hand_tracking: Switch, + #[schema(strings( display_name = "Prediction", help = r"Higher values make the controllers track smoother. @@ -700,6 +703,38 @@ Currently this cannot be reliably estimated automatically. The correct value sho pub haptics: Switch, } +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct HandTrackingConfig { + #[schema(flag = "real-time")] + #[schema(strings( + help = "Enabling this allows using hand gestures to emulate controller inputs." + ))] + pub use_gestures: Switch, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "Enabling this passes skeletal hand data (finger tracking) to SteamVR." + ))] + pub enable_skeleton: bool, +} + +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +pub struct HandGestureConfig { + #[schema(flag = "real-time")] + #[schema(strings( + help = "How close the tips of your fingers need to be to register a pinch." + ))] + #[schema(gui(slider(min = 0.05, max = 5.0)), suffix = "cm")] + pub pinch_touch_distance: f32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How close the tips of your fingers need to be to start registering a trigger pull." + ))] + #[schema(gui(slider(min = 0.05, max = 5.0)), suffix = "cm")] + pub pinch_trigger_distance: f32, +} + #[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy)] pub enum PositionRecenteringMode { Disabled, @@ -1232,6 +1267,21 @@ pub fn session_settings_default() -> SettingsDefault { }, force_threshold: 0.8, }, + hand_tracking: SwitchDefault { + enabled: true, + content: HandTrackingConfigDefault { + gui_collapsed: true, + use_gestures: SwitchDefault { + enabled: true, + content: HandGestureConfigDefault { + gui_collapsed: true, + pinch_touch_distance: 0.1, + pinch_trigger_distance: 1.0, + }, + }, + enable_skeleton: true, + }, + }, steamvr_pipeline_frames: 3.0, linear_velocity_cutoff: 0.05, angular_velocity_cutoff: 10.0, From a2176b0db7e8052263deed932fb2827eda0e2379 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:07:07 +0100 Subject: [PATCH 02/47] hands_to_gestures to get gestures from hand data --- alvr/server/src/tracking.rs | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 6488736f10..2b4a53f55f 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -325,6 +325,68 @@ pub fn to_openvr_hand_skeleton( ] } +#[derive(Debug, Copy, Clone)] +pub struct HandGesture { + pub active: bool, + pub touching: bool, + pub hover_dist: f32, +} + +pub fn hands_to_gestures( + config: &HeadsetConfig, + hand_skeleton: [Pose; 26], +) -> [HandGesture; 4] { + if let Switch::Enabled(controllers) = &config.controllers { + if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { + if let Switch::Enabled(use_gestures) = &hand_tracking.use_gestures { + // global joints + let gj = hand_skeleton; + + // if we model the tip of the finger as a spherical object, we should account for its radius + // these don't need to be configurable because they're accurate enough and the touch and trigger distances are already in settings + let thumb_rad: f32 = 0.01; // average thumb is ~20mm in diameter + let finger_rad: f32 = 0.0085; // average finger is ~17.5mm in diameter + + // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers + let pinch_min = use_gestures.pinch_touch_distance * 0.01 + thumb_rad + finger_rad; + let pinch_max = use_gestures.pinch_trigger_distance * 0.01 + thumb_rad + finger_rad; + + let thumb_tip: Pose = gj[5]; + let index_tip: Pose = gj[10]; + let middle_tip: Pose = gj[15]; + let ring_tip: Pose = gj[20]; + let little_tip: Pose = gj[25]; + + let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min; + let index_trigger = (1.0 - (thumb_tip.position.distance(index_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + + let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min; + let middle_trigger = (1.0 - (thumb_tip.position.distance(middle_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + + let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min; + let ring_trigger = (1.0 - (thumb_tip.position.distance(ring_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + + let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min; + let little_trigger = (1.0 - (thumb_tip.position.distance(little_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + + return [ + HandGesture { active: index_trigger > 0.0, touching: index_pinch, hover_dist: index_trigger }, + HandGesture { active: middle_trigger > 0.0, touching: middle_pinch, hover_dist: middle_trigger }, + HandGesture { active: ring_trigger > 0.0, touching: ring_pinch, hover_dist: ring_trigger }, + HandGesture { active: little_trigger > 0.0, touching: little_pinch, hover_dist: little_trigger }, + ]; + } + } + } + + [ + HandGesture { active: false, touching: false, hover_dist: 0.0 }, + HandGesture { active: false, touching: false, hover_dist: 0.0 }, + HandGesture { active: false, touching: false, hover_dist: 0.0 }, + HandGesture { active: false, touching: false, hover_dist: 0.0 }, + ] +} + pub fn to_ffi_motion(device_id: u64, motion: DeviceMotion) -> FfiDeviceMotion { FfiDeviceMotion { deviceID: device_id, From 83a9327a63d262cc59f9128c153be8ad299da0ed Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:07:49 +0100 Subject: [PATCH 03/47] activate button presses for gestures --- alvr/server/src/connection.rs | 127 +++++++++++++++++++++++++++++++--- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 8bcfb051b5..b9d26ea79b 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -5,7 +5,7 @@ use crate::{ input_mapping::ButtonMappingManager, sockets::WelcomeSocket, statistics::StatisticsManager, - tracking::{self, TrackingManager}, + tracking::{self, HandGesture, TrackingManager}, FfiFov, FfiViewsConfig, VideoPacket, BITRATE_MANAGER, DECODER_CONFIG, SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_MIRROR_SENDER, VIDEO_RECORDING_FILE, }; @@ -17,9 +17,12 @@ use alvr_common::{ once_cell::sync::Lazy, parking_lot::Mutex, settings_schema::Switch, - warn, AnyhowToCon, ConResult, ConnectionError, LazyMutOpt, RelaxedAtomic, ToCon, BUTTON_INFO, + warn, AnyhowToCon, ConResult, ConnectionError, LazyMutOpt, RelaxedAtomic, ToCon, BUTTON_INFO, A_CLICK_ID, B_CLICK_ID, CONTROLLER_PROFILE_INFO, DEVICE_ID_TO_PATH, HEAD_ID, LEFT_HAND_ID, - QUEST_CONTROLLER_PROFILE_PATH, RIGHT_HAND_ID, + QUEST_CONTROLLER_PROFILE_PATH, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, + LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_HAND_ID, + RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, + X_CLICK_ID, Y_CLICK_ID, }; use alvr_events::{ButtonEvent, EventType, HapticsEvent, TrackingEvent}; use alvr_packets::{ @@ -37,7 +40,7 @@ use std::{ io::Write, net::IpAddr, process::Command, - ptr, + ptr::{self}, sync::{ atomic::{AtomicBool, Ordering}, mpsc::{self, RecvTimeoutError, SyncSender, TrySendError}, @@ -671,14 +674,15 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { return; }; + let data_manager_lock = SERVER_DATA_MANAGER.read(); + let config = &data_manager_lock.settings().headset; + let mut tracking_manager_lock = tracking_manager.lock(); let motions; let left_hand_skeleton; let right_hand_skeleton; { - let data_manager_lock = SERVER_DATA_MANAGER.read(); - let config = &data_manager_lock.settings().headset; motions = tracking_manager_lock.transform_motions( config, &tracking.device_motions, @@ -703,7 +707,6 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .unwrap_or_default(); { - let data_manager_lock = SERVER_DATA_MANAGER.read(); if data_manager_lock.settings().logging.log_tracking { alvr_events::send_event(EventType::Tracking(Box::new(TrackingEvent { head_motion: motions @@ -748,6 +751,104 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { if let Some(stats) = &mut *STATISTICS_MANAGER.lock() { stats.report_tracking_received(tracking.target_timestamp); + // Handle gesture buttons + { + let mut did_disable_skeleton = false; + let mut press_gesture_buttons = + |gestures: [HandGesture; 4], binds: [[u64; 2]; 4]| { + for (i, g) in gestures.iter().enumerate() { + if g.active { + // Workaround for gestures not triggering button presses + if !did_disable_skeleton { + unsafe { + crate::SetTracking( + tracking.target_timestamp.as_nanos() as _, + stats.tracker_pose_time_offset().as_secs_f32(), + ffi_motions.as_ptr(), + ffi_motions.len() as _, + ptr::null(), + ptr::null(), + track_controllers, + ) + }; + did_disable_skeleton = true; + } + + // Handle touch bind + if binds[i][0] != 0 { + unsafe { + crate::SetButton( + binds[i][0], + crate::FfiButtonValue { + type_: + crate::FfiButtonType_BUTTON_TYPE_BINARY, + __bindgen_anon_1: + crate::FfiButtonValue__bindgen_ty_1 { + binary: g.touching.into(), + }, + }, + ) + } + } + // Handle hover bind + if binds[i][1] != 0 { + unsafe { + crate::SetButton( + binds[i][1], + crate::FfiButtonValue { + type_: + crate::FfiButtonType_BUTTON_TYPE_SCALAR, + __bindgen_anon_1: + crate::FfiButtonValue__bindgen_ty_1 { + scalar: g.hover_dist.into(), + }, + }, + ) + } + } + } + } + }; + + let hand_gestures = [ + tracking.hand_skeletons[0] + .map(|s| tracking::hands_to_gestures(config, s)), + tracking.hand_skeletons[1] + .map(|s| tracking::hands_to_gestures(config, s)), + ]; + + hand_gestures[0].map(|g| { + press_gesture_buttons( + g, + [ + [*LEFT_TRIGGER_CLICK_ID, *LEFT_TRIGGER_VALUE_ID], + [*Y_CLICK_ID, 0], + [*X_CLICK_ID, 0], + [*MENU_CLICK_ID, 0], + ], + ) + }); + + hand_gestures[1].map(|g| { + press_gesture_buttons( + g, + [ + [*RIGHT_TRIGGER_CLICK_ID, *RIGHT_TRIGGER_VALUE_ID], + [*B_CLICK_ID, 0], + [*A_CLICK_ID, 0], + [0, 0], + ], + ) + }); + } + + let mut hand_skeletons_enabled = false; + if let Switch::Enabled(controllers) = &config.controllers { + if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { + hand_skeletons_enabled = hand_tracking.enable_skeleton; + } + } + unsafe { crate::SetTracking( tracking.target_timestamp.as_nanos() as _, @@ -755,12 +856,20 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { ffi_motions.as_ptr(), ffi_motions.len() as _, if let Some(skeleton) = &ffi_left_hand_skeleton { - skeleton + if hand_skeletons_enabled { + skeleton + } else { + ptr::null() + } } else { ptr::null() }, if let Some(skeleton) = &ffi_right_hand_skeleton { - skeleton + if hand_skeletons_enabled { + skeleton + } else { + ptr::null() + } } else { ptr::null() }, From 3ba7909d6d166da532ee1e2fb6c8f10cfa7f0003 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:28:20 +0100 Subject: [PATCH 04/47] adjust some values to improve gesture accuracy --- alvr/server/src/tracking.rs | 37 +++++++++++++++++++----------------- alvr/session/src/settings.rs | 8 ++++---- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 2b4a53f55f..e158c10a7d 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -343,13 +343,16 @@ pub fn hands_to_gestures( let gj = hand_skeleton; // if we model the tip of the finger as a spherical object, we should account for its radius - // these don't need to be configurable because they're accurate enough and the touch and trigger distances are already in settings - let thumb_rad: f32 = 0.01; // average thumb is ~20mm in diameter - let finger_rad: f32 = 0.0085; // average finger is ~17.5mm in diameter + // these are intentionally under the average by ~5mm since the touch and trigger distances are already configurable in settings + let thumb_rad: f32 = 0.0075; // average thumb is ~20mm in diameter + let index_rad: f32 = 0.006; // average index finger is ~17mm in diameter + let middle_rad: f32 = 0.006; // average middle finger is ~17mm in diameter + let ring_rad: f32 = 0.0055; // average ring finger is ~16mm in diameter + let little_rad: f32 = 0.0045; // average pinky finger is ~14mm in diameter // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers - let pinch_min = use_gestures.pinch_touch_distance * 0.01 + thumb_rad + finger_rad; - let pinch_max = use_gestures.pinch_trigger_distance * 0.01 + thumb_rad + finger_rad; + let pinch_min = use_gestures.pinch_touch_distance * 0.01; + let pinch_max = use_gestures.pinch_trigger_distance * 0.01; let thumb_tip: Pose = gj[5]; let index_tip: Pose = gj[10]; @@ -357,23 +360,23 @@ pub fn hands_to_gestures( let ring_tip: Pose = gj[20]; let little_tip: Pose = gj[25]; - let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min; - let index_trigger = (1.0 - (thumb_tip.position.distance(index_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; + let index_trigger = (1.0 - (thumb_tip.position.distance(index_tip.position) - pinch_min - thumb_rad - ring_rad)/pinch_max + thumb_rad + index_rad).clamp(0.0, 1.0); - let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min; - let middle_trigger = (1.0 - (thumb_tip.position.distance(middle_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; + let middle_trigger = (1.0 - (thumb_tip.position.distance(middle_tip.position) - pinch_min - thumb_rad - middle_rad)/pinch_max + thumb_rad + middle_rad).clamp(0.0, 1.0); - let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min; - let ring_trigger = (1.0 - (thumb_tip.position.distance(ring_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; + let ring_trigger = (1.0 - (thumb_tip.position.distance(ring_tip.position) - pinch_min - thumb_rad - ring_rad)/pinch_max + thumb_rad + ring_rad).clamp(0.0, 1.0); - let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min; - let little_trigger = (1.0 - (thumb_tip.position.distance(little_tip.position) - pinch_min)/pinch_max).clamp(0.0, 1.0); + let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; + let little_trigger = (1.0 - (thumb_tip.position.distance(little_tip.position) - pinch_min - thumb_rad - little_rad)/pinch_max + thumb_rad + little_rad).clamp(0.0, 1.0); return [ - HandGesture { active: index_trigger > 0.0, touching: index_pinch, hover_dist: index_trigger }, - HandGesture { active: middle_trigger > 0.0, touching: middle_pinch, hover_dist: middle_trigger }, - HandGesture { active: ring_trigger > 0.0, touching: ring_pinch, hover_dist: ring_trigger }, - HandGesture { active: little_trigger > 0.0, touching: little_pinch, hover_dist: little_trigger }, + HandGesture { active: true, touching: index_pinch, hover_dist: index_trigger }, + HandGesture { active: true, touching: middle_pinch, hover_dist: middle_trigger }, + HandGesture { active: true, touching: ring_pinch, hover_dist: ring_trigger }, + HandGesture { active: true, touching: little_pinch, hover_dist: little_trigger }, ]; } } diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 831a1c7b4a..a08e574880 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -724,14 +724,14 @@ pub struct HandGestureConfig { #[schema(strings( help = "How close the tips of your fingers need to be to register a pinch." ))] - #[schema(gui(slider(min = 0.05, max = 5.0)), suffix = "cm")] + #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)), suffix = "cm")] pub pinch_touch_distance: f32, #[schema(flag = "real-time")] #[schema(strings( help = "How close the tips of your fingers need to be to start registering a trigger pull." ))] - #[schema(gui(slider(min = 0.05, max = 5.0)), suffix = "cm")] + #[schema(gui(slider(min = 0.00, max = 2.5, step = 0.025)), suffix = "cm")] pub pinch_trigger_distance: f32, } @@ -1275,8 +1275,8 @@ pub fn session_settings_default() -> SettingsDefault { enabled: true, content: HandGestureConfigDefault { gui_collapsed: true, - pinch_touch_distance: 0.1, - pinch_trigger_distance: 1.0, + pinch_touch_distance: 0.0, + pinch_trigger_distance: 0.25, }, }, enable_skeleton: true, From e00abc89c2ba20a2fc9d212d4afbc3534bfdb654 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:35:23 +0100 Subject: [PATCH 05/47] move gesture binds into hands_to_gestures --- alvr/server/src/connection.rs | 39 +++--------- alvr/server/src/tracking.rs | 117 ++++++++++++++++++++++++++++------ 2 files changed, 107 insertions(+), 49 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index b9d26ea79b..1ed08f6193 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -755,7 +755,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { { let mut did_disable_skeleton = false; let mut press_gesture_buttons = - |gestures: [HandGesture; 4], binds: [[u64; 2]; 4]| { + |gestures: [HandGesture; 4]| { for (i, g) in gestures.iter().enumerate() { if g.active { // Workaround for gestures not triggering button presses @@ -775,10 +775,10 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { } // Handle touch bind - if binds[i][0] != 0 { + if g.touch_bind != 0 { unsafe { crate::SetButton( - binds[i][0], + g.touch_bind, crate::FfiButtonValue { type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, @@ -791,10 +791,10 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { } } // Handle hover bind - if binds[i][1] != 0 { + if g.hover_bind != 0 { unsafe { crate::SetButton( - binds[i][1], + g.hover_bind, crate::FfiButtonValue { type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, @@ -812,34 +812,13 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { let hand_gestures = [ tracking.hand_skeletons[0] - .map(|s| tracking::hands_to_gestures(config, s)), + .map(|s| tracking::hands_to_gestures(config, *LEFT_HAND_ID, s)), tracking.hand_skeletons[1] - .map(|s| tracking::hands_to_gestures(config, s)), + .map(|s| tracking::hands_to_gestures(config, *RIGHT_HAND_ID, s)), ]; - hand_gestures[0].map(|g| { - press_gesture_buttons( - g, - [ - [*LEFT_TRIGGER_CLICK_ID, *LEFT_TRIGGER_VALUE_ID], - [*Y_CLICK_ID, 0], - [*X_CLICK_ID, 0], - [*MENU_CLICK_ID, 0], - ], - ) - }); - - hand_gestures[1].map(|g| { - press_gesture_buttons( - g, - [ - [*RIGHT_TRIGGER_CLICK_ID, *RIGHT_TRIGGER_VALUE_ID], - [*B_CLICK_ID, 0], - [*A_CLICK_ID, 0], - [0, 0], - ], - ) - }); + hand_gestures[0].map(press_gesture_buttons); + hand_gestures[1].map(press_gesture_buttons); } let mut hand_skeletons_enabled = false; diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index e158c10a7d..f471b21947 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -330,10 +330,13 @@ pub struct HandGesture { pub active: bool, pub touching: bool, pub hover_dist: f32, + pub touch_bind: u64, + pub hover_bind: u64, } pub fn hands_to_gestures( config: &HeadsetConfig, + device_id: u64, hand_skeleton: [Pose; 26], ) -> [HandGesture; 4] { if let Switch::Enabled(controllers) = &config.controllers { @@ -360,33 +363,109 @@ pub fn hands_to_gestures( let ring_tip: Pose = gj[20]; let little_tip: Pose = gj[25]; - let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; - let index_trigger = (1.0 - (thumb_tip.position.distance(index_tip.position) - pinch_min - thumb_rad - ring_rad)/pinch_max + thumb_rad + index_rad).clamp(0.0, 1.0); - - let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; - let middle_trigger = (1.0 - (thumb_tip.position.distance(middle_tip.position) - pinch_min - thumb_rad - middle_rad)/pinch_max + thumb_rad + middle_rad).clamp(0.0, 1.0); - - let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; - let ring_trigger = (1.0 - (thumb_tip.position.distance(ring_tip.position) - pinch_min - thumb_rad - ring_rad)/pinch_max + thumb_rad + ring_rad).clamp(0.0, 1.0); - - let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; - let little_trigger = (1.0 - (thumb_tip.position.distance(little_tip.position) - pinch_min - thumb_rad - little_rad)/pinch_max + thumb_rad + little_rad).clamp(0.0, 1.0); + let index_pinch = thumb_tip.position.distance(index_tip.position) + < pinch_min + thumb_rad + index_rad; + let index_trigger = (1.0 + - (thumb_tip.position.distance(index_tip.position) + - pinch_min + - thumb_rad + - ring_rad) + / pinch_max + + thumb_rad + + index_rad) + .clamp(0.0, 1.0); + + let middle_pinch = thumb_tip.position.distance(middle_tip.position) + < pinch_min + thumb_rad + middle_rad; + let middle_trigger = (1.0 + - (thumb_tip.position.distance(middle_tip.position) + - pinch_min + - thumb_rad + - middle_rad) + / pinch_max + + thumb_rad + + middle_rad) + .clamp(0.0, 1.0); + + let ring_pinch = thumb_tip.position.distance(ring_tip.position) + < pinch_min + thumb_rad + ring_rad; + let ring_trigger = (1.0 + - (thumb_tip.position.distance(ring_tip.position) + - pinch_min + - thumb_rad + - ring_rad) + / pinch_max + + thumb_rad + + ring_rad) + .clamp(0.0, 1.0); + + let little_pinch = thumb_tip.position.distance(little_tip.position) + < pinch_min + thumb_rad + little_rad; + let little_trigger = (1.0 + - (thumb_tip.position.distance(little_tip.position) + - pinch_min + - thumb_rad + - little_rad) + / pinch_max + + thumb_rad + + little_rad) + .clamp(0.0, 1.0); return [ - HandGesture { active: true, touching: index_pinch, hover_dist: index_trigger }, - HandGesture { active: true, touching: middle_pinch, hover_dist: middle_trigger }, - HandGesture { active: true, touching: ring_pinch, hover_dist: ring_trigger }, - HandGesture { active: true, touching: little_pinch, hover_dist: little_trigger }, + HandGesture { + active: true, + touching: index_pinch, + hover_dist: index_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_CLICK_ID } else { *RIGHT_TRIGGER_CLICK_ID }, + hover_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_VALUE_ID } else { *RIGHT_TRIGGER_VALUE_ID }, + }, + HandGesture { + active: true, + touching: middle_pinch, + hover_dist: middle_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { *Y_CLICK_ID } else { 0 }, + hover_bind: if device_id == *LEFT_HAND_ID { *B_CLICK_ID } else { 0 }, + }, + HandGesture { + active: true, + touching: ring_pinch, + hover_dist: ring_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { *X_CLICK_ID } else { 0 }, + hover_bind: if device_id == *LEFT_HAND_ID { *A_CLICK_ID } else { 0 }, + }, + HandGesture { + active: true, + touching: little_pinch, + hover_dist: little_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { *MENU_CLICK_ID } else { 0 }, + hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, + }, ]; } } } [ - HandGesture { active: false, touching: false, hover_dist: 0.0 }, - HandGesture { active: false, touching: false, hover_dist: 0.0 }, - HandGesture { active: false, touching: false, hover_dist: 0.0 }, - HandGesture { active: false, touching: false, hover_dist: 0.0 }, + HandGesture { + active: false, + touching: false, + hover_dist: 0.0, + }, + HandGesture { + active: false, + touching: false, + hover_dist: 0.0, + }, + HandGesture { + active: false, + touching: false, + hover_dist: 0.0, + }, + HandGesture { + active: false, + touching: false, + hover_dist: 0.0, + }, ] } From c28461df3bfed362af50b913aa4a90b9039d15ca Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:36:23 +0100 Subject: [PATCH 06/47] remember to set empty binds on dummy list --- alvr/server/src/tracking.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index f471b21947..b0f8ddaf28 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -450,21 +450,29 @@ pub fn hands_to_gestures( active: false, touching: false, hover_dist: 0.0, + touch_bind: 0, + hover_bind: 0, }, HandGesture { active: false, touching: false, hover_dist: 0.0, + touch_bind: 0, + hover_bind: 0, }, HandGesture { active: false, touching: false, hover_dist: 0.0, + touch_bind: 0, + hover_bind: 0, }, HandGesture { active: false, touching: false, hover_dist: 0.0, + touch_bind: 0, + hover_bind: 0, }, ] } From bc52852c0f32234f672a093c79ee9b6fc72d85a0 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:38:34 +0100 Subject: [PATCH 07/47] fix build issues --- alvr/server/src/connection.rs | 11 ++++------- alvr/server/src/tracking.rs | 3 +++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 1ed08f6193..6465106dbe 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -17,12 +17,9 @@ use alvr_common::{ once_cell::sync::Lazy, parking_lot::Mutex, settings_schema::Switch, - warn, AnyhowToCon, ConResult, ConnectionError, LazyMutOpt, RelaxedAtomic, ToCon, BUTTON_INFO, A_CLICK_ID, B_CLICK_ID, + warn, AnyhowToCon, ConResult, ConnectionError, LazyMutOpt, RelaxedAtomic, ToCon, BUTTON_INFO, CONTROLLER_PROFILE_INFO, DEVICE_ID_TO_PATH, HEAD_ID, LEFT_HAND_ID, - QUEST_CONTROLLER_PROFILE_PATH, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, - LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_HAND_ID, - RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, - X_CLICK_ID, Y_CLICK_ID, + QUEST_CONTROLLER_PROFILE_PATH, RIGHT_HAND_ID, }; use alvr_events::{ButtonEvent, EventType, HapticsEvent, TrackingEvent}; use alvr_packets::{ @@ -817,8 +814,8 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .map(|s| tracking::hands_to_gestures(config, *RIGHT_HAND_ID, s)), ]; - hand_gestures[0].map(press_gesture_buttons); - hand_gestures[1].map(press_gesture_buttons); + hand_gestures[0].map(|g| press_gesture_buttons(g)); + hand_gestures[1].map(|g| press_gesture_buttons(g)); } let mut hand_skeletons_enabled = false; diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index b0f8ddaf28..a540bd9250 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -2,6 +2,9 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ glam::{EulerRot, Quat, Vec3}, DeviceMotion, Pose, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID, + LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, + RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, + A_CLICK_ID, B_CLICK_ID, X_CLICK_ID, Y_CLICK_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, From 82972aefb6061f2eca0234eed18dc22b43454fc0 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:39:18 +0100 Subject: [PATCH 08/47] remove unnecessary enumerate --- alvr/server/src/connection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 6465106dbe..8ccdc9b5dc 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -753,7 +753,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { let mut did_disable_skeleton = false; let mut press_gesture_buttons = |gestures: [HandGesture; 4]| { - for (i, g) in gestures.iter().enumerate() { + for g in gestures { if g.active { // Workaround for gestures not triggering button presses if !did_disable_skeleton { From 7af35de09769cbc67f0ec66f7b760d611f44e37a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 01:07:42 +0100 Subject: [PATCH 09/47] support curl and add grip gesture --- alvr/server/src/connection.rs | 4 +- alvr/server/src/tracking.rs | 116 ++++++++++++++++++++++++++++------ 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 8ccdc9b5dc..5145faec78 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -752,7 +752,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { { let mut did_disable_skeleton = false; let mut press_gesture_buttons = - |gestures: [HandGesture; 4]| { + |gestures: [HandGesture; 5]| { for g in gestures { if g.active { // Workaround for gestures not triggering button presses @@ -797,7 +797,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { crate::FfiButtonType_BUTTON_TYPE_SCALAR, __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - scalar: g.hover_dist.into(), + scalar: g.hover_val.into(), }, }, ) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index a540bd9250..8a1b878ea5 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -1,10 +1,10 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ glam::{EulerRot, Quat, Vec3}, - DeviceMotion, Pose, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID, - LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, - RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, - A_CLICK_ID, B_CLICK_ID, X_CLICK_ID, Y_CLICK_ID, + warn, DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, + LEFT_SQUEEZE_VALUE_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, + RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_TRIGGER_CLICK_ID, + RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, Y_CLICK_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, @@ -332,7 +332,7 @@ pub fn to_openvr_hand_skeleton( pub struct HandGesture { pub active: bool, pub touching: bool, - pub hover_dist: f32, + pub hover_val: f32, pub touch_bind: u64, pub hover_bind: u64, } @@ -341,7 +341,7 @@ pub fn hands_to_gestures( config: &HeadsetConfig, device_id: u64, hand_skeleton: [Pose; 26], -) -> [HandGesture; 4] { +) -> [HandGesture; 5] { if let Switch::Enabled(controllers) = &config.controllers { if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { if let Switch::Enabled(use_gestures) = &hand_tracking.use_gestures { @@ -360,12 +360,20 @@ pub fn hands_to_gestures( let pinch_min = use_gestures.pinch_touch_distance * 0.01; let pinch_max = use_gestures.pinch_trigger_distance * 0.01; + let palm: Pose = gj[0]; let thumb_tip: Pose = gj[5]; + let index_metacarpal: Pose = gj[6]; let index_tip: Pose = gj[10]; + let middle_metacarpal: Pose = gj[11]; let middle_tip: Pose = gj[15]; + let ring_metacarpal: Pose = gj[16]; let ring_tip: Pose = gj[20]; + let little_metacarpal: Pose = gj[21]; let little_tip: Pose = gj[25]; + let thumb_curl = palm.position.distance(thumb_tip.position); + warn!("thumb_curl: {}", thumb_curl); + let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; let index_trigger = (1.0 @@ -378,6 +386,9 @@ pub fn hands_to_gestures( + index_rad) .clamp(0.0, 1.0); + let index_curl = index_metacarpal.position.distance(index_tip.position); + warn!("index_curl: {}", index_curl); + let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; let middle_trigger = (1.0 @@ -390,6 +401,9 @@ pub fn hands_to_gestures( + middle_rad) .clamp(0.0, 1.0); + let middle_curl = middle_metacarpal.position.distance(middle_tip.position); + warn!("middle_curl: {}", middle_curl); + let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; let ring_trigger = (1.0 @@ -402,6 +416,9 @@ pub fn hands_to_gestures( + ring_rad) .clamp(0.0, 1.0); + let ring_curl = ring_metacarpal.position.distance(ring_tip.position); + warn!("ring_curl: {}", ring_curl); + let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; let little_trigger = (1.0 @@ -414,35 +431,85 @@ pub fn hands_to_gestures( + little_rad) .clamp(0.0, 1.0); + let little_curl = little_metacarpal.position.distance(little_tip.position); + warn!("little_curl: {}", little_curl); + + let grip_curl = (1.0 + - ((middle_curl + ring_curl + little_curl) / 3.0 - 0.05) * 20.0) + .clamp(0.0, 1.0); + return [ HandGesture { active: true, touching: index_pinch, - hover_dist: index_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_CLICK_ID } else { *RIGHT_TRIGGER_CLICK_ID }, - hover_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_VALUE_ID } else { *RIGHT_TRIGGER_VALUE_ID }, + hover_val: index_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { + *LEFT_TRIGGER_CLICK_ID + } else { + *RIGHT_TRIGGER_CLICK_ID + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *LEFT_TRIGGER_VALUE_ID + } else { + *RIGHT_TRIGGER_VALUE_ID + }, }, HandGesture { active: true, touching: middle_pinch, - hover_dist: middle_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { *Y_CLICK_ID } else { 0 }, - hover_bind: if device_id == *LEFT_HAND_ID { *B_CLICK_ID } else { 0 }, + hover_val: middle_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { + *Y_CLICK_ID + } else { + 0 + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *B_CLICK_ID + } else { + 0 + }, }, HandGesture { active: true, touching: ring_pinch, - hover_dist: ring_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { *X_CLICK_ID } else { 0 }, - hover_bind: if device_id == *LEFT_HAND_ID { *A_CLICK_ID } else { 0 }, + hover_val: ring_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { + *X_CLICK_ID + } else { + 0 + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *A_CLICK_ID + } else { + 0 + }, }, HandGesture { active: true, touching: little_pinch, - hover_dist: little_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { *MENU_CLICK_ID } else { 0 }, + hover_val: little_trigger, + touch_bind: if device_id == *LEFT_HAND_ID { + *MENU_CLICK_ID + } else { + 0 + }, hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, }, + HandGesture { + active: true, + touching: grip_curl == 1.0, + hover_val: grip_curl, + touch_bind: if device_id == *LEFT_HAND_ID { + *LEFT_SQUEEZE_CLICK_ID + } else { + *RIGHT_SQUEEZE_CLICK_ID + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *LEFT_SQUEEZE_VALUE_ID + } else { + *RIGHT_SQUEEZE_VALUE_ID + }, + }, ]; } } @@ -452,28 +519,35 @@ pub fn hands_to_gestures( HandGesture { active: false, touching: false, - hover_dist: 0.0, + hover_val: 0.0, + touch_bind: 0, + hover_bind: 0, + }, + HandGesture { + active: false, + touching: false, + hover_val: 0.0, touch_bind: 0, hover_bind: 0, }, HandGesture { active: false, touching: false, - hover_dist: 0.0, + hover_val: 0.0, touch_bind: 0, hover_bind: 0, }, HandGesture { active: false, touching: false, - hover_dist: 0.0, + hover_val: 0.0, touch_bind: 0, hover_bind: 0, }, HandGesture { active: false, touching: false, - hover_dist: 0.0, + hover_val: 0.0, touch_bind: 0, hover_bind: 0, }, From 3a95b26f33fde592c762c3e10a6301f2c0758238 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 01:13:21 +0100 Subject: [PATCH 10/47] make finger thresholds slightly larger --- alvr/server/src/tracking.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 8a1b878ea5..667fdaaa72 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -351,10 +351,10 @@ pub fn hands_to_gestures( // if we model the tip of the finger as a spherical object, we should account for its radius // these are intentionally under the average by ~5mm since the touch and trigger distances are already configurable in settings let thumb_rad: f32 = 0.0075; // average thumb is ~20mm in diameter - let index_rad: f32 = 0.006; // average index finger is ~17mm in diameter - let middle_rad: f32 = 0.006; // average middle finger is ~17mm in diameter - let ring_rad: f32 = 0.0055; // average ring finger is ~16mm in diameter - let little_rad: f32 = 0.0045; // average pinky finger is ~14mm in diameter + let index_rad: f32 = 0.0065; // average index finger is ~18mm in diameter + let middle_rad: f32 = 0.0065; // average middle finger is ~18mm in diameter + let ring_rad: f32 = 0.006; // average ring finger is ~17mm in diameter + let little_rad: f32 = 0.005; // average pinky finger is ~15mm in diameter // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers let pinch_min = use_gestures.pinch_touch_distance * 0.01; From d6727c1f173a7d51af2015fdf2c77ff5cee5d22f Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:40:43 +0100 Subject: [PATCH 11/47] add curl distance settings --- alvr/server/src/tracking.rs | 67 +++++++++++++++++++++++++++++------- alvr/session/src/settings.rs | 16 +++++++++ 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 667fdaaa72..7d20759d3b 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -355,23 +355,34 @@ pub fn hands_to_gestures( let middle_rad: f32 = 0.0065; // average middle finger is ~18mm in diameter let ring_rad: f32 = 0.006; // average ring finger is ~17mm in diameter let little_rad: f32 = 0.005; // average pinky finger is ~15mm in diameter + let palm_depth: f32 = 0.005; // average palm bones are ~10mm from the skin // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers let pinch_min = use_gestures.pinch_touch_distance * 0.01; let pinch_max = use_gestures.pinch_trigger_distance * 0.01; + let curl_min = use_gestures.curl_touch_distance * 0.01; + let curl_max = use_gestures.curl_trigger_distance * 0.01; let palm: Pose = gj[0]; let thumb_tip: Pose = gj[5]; - let index_metacarpal: Pose = gj[6]; + let index_proximal: Pose = gj[7]; let index_tip: Pose = gj[10]; - let middle_metacarpal: Pose = gj[11]; + let middle_proximal: Pose = gj[12]; let middle_tip: Pose = gj[15]; - let ring_metacarpal: Pose = gj[16]; + let ring_proximal: Pose = gj[17]; let ring_tip: Pose = gj[20]; - let little_metacarpal: Pose = gj[21]; + let little_proximal: Pose = gj[22]; let little_tip: Pose = gj[25]; - let thumb_curl = palm.position.distance(thumb_tip.position); + let thumb_curl = (1.0 + - (palm.position.distance(thumb_tip.position) + - curl_min + - palm_depth + - thumb_rad) + / curl_max + + palm_depth + + thumb_rad) + .clamp(0.0, 1.0); warn!("thumb_curl: {}", thumb_curl); let index_pinch = thumb_tip.position.distance(index_tip.position) @@ -380,13 +391,21 @@ pub fn hands_to_gestures( - (thumb_tip.position.distance(index_tip.position) - pinch_min - thumb_rad - - ring_rad) + - index_rad) / pinch_max + thumb_rad + index_rad) .clamp(0.0, 1.0); - let index_curl = index_metacarpal.position.distance(index_tip.position); + let index_curl = (1.0 + - (index_proximal.position.distance(index_tip.position) + - curl_min + - palm_depth + - index_rad) + / curl_max + + palm_depth + + index_rad) + .clamp(0.0, 1.0); warn!("index_curl: {}", index_curl); let middle_pinch = thumb_tip.position.distance(middle_tip.position) @@ -401,7 +420,15 @@ pub fn hands_to_gestures( + middle_rad) .clamp(0.0, 1.0); - let middle_curl = middle_metacarpal.position.distance(middle_tip.position); + let middle_curl = (1.0 + - (middle_proximal.position.distance(middle_tip.position) + - curl_min + - palm_depth + - middle_rad) + / curl_max + + palm_depth + + middle_rad) + .clamp(0.0, 1.0); warn!("middle_curl: {}", middle_curl); let ring_pinch = thumb_tip.position.distance(ring_tip.position) @@ -416,7 +443,15 @@ pub fn hands_to_gestures( + ring_rad) .clamp(0.0, 1.0); - let ring_curl = ring_metacarpal.position.distance(ring_tip.position); + let ring_curl = (1.0 + - (ring_proximal.position.distance(ring_tip.position) + - curl_min + - palm_depth + - ring_rad) + / curl_max + + palm_depth + + ring_rad) + .clamp(0.0, 1.0); warn!("ring_curl: {}", ring_curl); let little_pinch = thumb_tip.position.distance(little_tip.position) @@ -431,12 +466,18 @@ pub fn hands_to_gestures( + little_rad) .clamp(0.0, 1.0); - let little_curl = little_metacarpal.position.distance(little_tip.position); + let little_curl = (1.0 + - (little_proximal.position.distance(little_tip.position) + - curl_min + - palm_depth + - little_rad) + / curl_max + + palm_depth + + little_rad) + .clamp(0.0, 1.0); warn!("little_curl: {}", little_curl); - let grip_curl = (1.0 - - ((middle_curl + ring_curl + little_curl) / 3.0 - 0.05) * 20.0) - .clamp(0.0, 1.0); + let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; return [ HandGesture { diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index a08e574880..126699f484 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -733,6 +733,20 @@ pub struct HandGestureConfig { ))] #[schema(gui(slider(min = 0.00, max = 2.5, step = 0.025)), suffix = "cm")] pub pinch_trigger_distance: f32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How curled your fingers need to be to register a click." + ))] + #[schema(gui(slider(min = 0.0, max = 5.0)), suffix = "cm")] + pub curl_touch_distance: f32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How curled your fingers need to be to start registering a trigger pull." + ))] + #[schema(gui(slider(min = 0.0, max = 10.0)), suffix = "cm")] + pub curl_trigger_distance: f32, } #[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy)] @@ -1277,6 +1291,8 @@ pub fn session_settings_default() -> SettingsDefault { gui_collapsed: true, pinch_touch_distance: 0.0, pinch_trigger_distance: 0.25, + curl_touch_distance: 2.0, + curl_trigger_distance: 2.5, }, }, enable_skeleton: true, From c574af6d6e6b22fb36602d105c70c9f628be3093 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:51:37 +0100 Subject: [PATCH 12/47] curl: use min of proximal and metacarpal --- alvr/server/src/tracking.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 7d20759d3b..43f07a1a01 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -365,12 +365,16 @@ pub fn hands_to_gestures( let palm: Pose = gj[0]; let thumb_tip: Pose = gj[5]; + let index_metacarpal: Pose = gj[6]; let index_proximal: Pose = gj[7]; let index_tip: Pose = gj[10]; + let middle_metacarpal: Pose = gj[11]; let middle_proximal: Pose = gj[12]; let middle_tip: Pose = gj[15]; + let ring_metacarpal: Pose = gj[16]; let ring_proximal: Pose = gj[17]; let ring_tip: Pose = gj[20]; + let little_metacarpal: Pose = gj[21]; let little_proximal: Pose = gj[22]; let little_tip: Pose = gj[25]; @@ -398,7 +402,7 @@ pub fn hands_to_gestures( .clamp(0.0, 1.0); let index_curl = (1.0 - - (index_proximal.position.distance(index_tip.position) + - (index_metacarpal.position.distance(index_tip.position).min(index_proximal.position.distance(index_tip.position)) - curl_min - palm_depth - index_rad) @@ -421,7 +425,7 @@ pub fn hands_to_gestures( .clamp(0.0, 1.0); let middle_curl = (1.0 - - (middle_proximal.position.distance(middle_tip.position) + - (middle_metacarpal.position.distance(middle_tip.position).min(middle_proximal.position.distance(middle_tip.position)) - curl_min - palm_depth - middle_rad) @@ -444,7 +448,7 @@ pub fn hands_to_gestures( .clamp(0.0, 1.0); let ring_curl = (1.0 - - (ring_proximal.position.distance(ring_tip.position) + - (ring_metacarpal.position.distance(ring_tip.position).min(ring_proximal.position.distance(ring_tip.position)) - curl_min - palm_depth - ring_rad) @@ -467,7 +471,7 @@ pub fn hands_to_gestures( .clamp(0.0, 1.0); let little_curl = (1.0 - - (little_proximal.position.distance(little_tip.position) + - (little_metacarpal.position.distance(little_tip.position).min(little_proximal.position.distance(little_tip.position)) - curl_min - palm_depth - little_rad) From 0a00862781920a2afe11a62ab4224b79dd8e305e Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:15:44 +0100 Subject: [PATCH 13/47] add joystick press gesture + correct curl maths --- alvr/server/src/connection.rs | 2 +- alvr/server/src/tracking.rs | 83 ++++++++++++++++++++--------------- alvr/session/src/settings.rs | 2 +- 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 5145faec78..096b923e22 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -752,7 +752,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { { let mut did_disable_skeleton = false; let mut press_gesture_buttons = - |gestures: [HandGesture; 5]| { + |gestures: [HandGesture; 6]| { for g in gestures { if g.active { // Workaround for gestures not triggering button presses diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 43f07a1a01..5cc00a3fc5 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -2,9 +2,10 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ glam::{EulerRot, Quat, Vec3}, warn, DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, - LEFT_SQUEEZE_VALUE_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, - RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_TRIGGER_CLICK_ID, - RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, Y_CLICK_ID, + LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, + MENU_CLICK_ID, RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, + RIGHT_THUMBSTICK_CLICK_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, + Y_CLICK_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, @@ -341,7 +342,7 @@ pub fn hands_to_gestures( config: &HeadsetConfig, device_id: u64, hand_skeleton: [Pose; 26], -) -> [HandGesture; 5] { +) -> [HandGesture; 6] { if let Switch::Enabled(controllers) = &config.controllers { if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { if let Switch::Enabled(use_gestures) = &hand_tracking.use_gestures { @@ -383,9 +384,7 @@ pub fn hands_to_gestures( - curl_min - palm_depth - thumb_rad) - / curl_max - + palm_depth - + thumb_rad) + / (curl_max + palm_depth + thumb_rad)) .clamp(0.0, 1.0); warn!("thumb_curl: {}", thumb_curl); @@ -396,19 +395,18 @@ pub fn hands_to_gestures( - pinch_min - thumb_rad - index_rad) - / pinch_max - + thumb_rad - + index_rad) + / (pinch_max + thumb_rad + index_rad)) .clamp(0.0, 1.0); let index_curl = (1.0 - - (index_metacarpal.position.distance(index_tip.position).min(index_proximal.position.distance(index_tip.position)) + - (index_metacarpal + .position + .lerp(index_proximal.position, 0.5) + .distance(index_tip.position) - curl_min - palm_depth - index_rad) - / curl_max - + palm_depth - + index_rad) + / (curl_max + palm_depth + index_rad)) .clamp(0.0, 1.0); warn!("index_curl: {}", index_curl); @@ -419,19 +417,18 @@ pub fn hands_to_gestures( - pinch_min - thumb_rad - middle_rad) - / pinch_max - + thumb_rad - + middle_rad) + / (pinch_max + thumb_rad + middle_rad)) .clamp(0.0, 1.0); let middle_curl = (1.0 - - (middle_metacarpal.position.distance(middle_tip.position).min(middle_proximal.position.distance(middle_tip.position)) + - (middle_metacarpal + .position + .lerp(middle_proximal.position, 0.5) + .distance(middle_tip.position) - curl_min - palm_depth - middle_rad) - / curl_max - + palm_depth - + middle_rad) + / (curl_max + palm_depth + middle_rad)) .clamp(0.0, 1.0); warn!("middle_curl: {}", middle_curl); @@ -442,19 +439,18 @@ pub fn hands_to_gestures( - pinch_min - thumb_rad - ring_rad) - / pinch_max - + thumb_rad - + ring_rad) + / (pinch_max + thumb_rad + ring_rad)) .clamp(0.0, 1.0); let ring_curl = (1.0 - - (ring_metacarpal.position.distance(ring_tip.position).min(ring_proximal.position.distance(ring_tip.position)) + - (ring_metacarpal + .position + .lerp(ring_proximal.position, 0.5) + .distance(ring_tip.position) - curl_min - palm_depth - ring_rad) - / curl_max - + palm_depth - + ring_rad) + / (curl_max + palm_depth + ring_rad)) .clamp(0.0, 1.0); warn!("ring_curl: {}", ring_curl); @@ -465,19 +461,18 @@ pub fn hands_to_gestures( - pinch_min - thumb_rad - little_rad) - / pinch_max - + thumb_rad - + little_rad) + / (pinch_max + thumb_rad + little_rad)) .clamp(0.0, 1.0); let little_curl = (1.0 - - (little_metacarpal.position.distance(little_tip.position).min(little_proximal.position.distance(little_tip.position)) + - (little_metacarpal + .position + .lerp(little_proximal.position, 0.5) + .distance(little_tip.position) - curl_min - palm_depth - little_rad) - / curl_max - + palm_depth - + little_rad) + / (curl_max + palm_depth + little_rad)) .clamp(0.0, 1.0); warn!("little_curl: {}", little_curl); @@ -555,6 +550,17 @@ pub fn hands_to_gestures( *RIGHT_SQUEEZE_VALUE_ID }, }, + HandGesture { + active: true, + touching: thumb_curl == 1.0, + hover_val: thumb_curl, + touch_bind: if device_id == *LEFT_HAND_ID { + *LEFT_THUMBSTICK_CLICK_ID + } else { + *RIGHT_THUMBSTICK_CLICK_ID + }, + hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, + }, ]; } } @@ -596,6 +602,13 @@ pub fn hands_to_gestures( touch_bind: 0, hover_bind: 0, }, + HandGesture { + active: false, + touching: false, + hover_val: 0.0, + touch_bind: 0, + hover_bind: 0, + }, ] } diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 126699f484..5f5e7cbc20 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -731,7 +731,7 @@ pub struct HandGestureConfig { #[schema(strings( help = "How close the tips of your fingers need to be to start registering a trigger pull." ))] - #[schema(gui(slider(min = 0.00, max = 2.5, step = 0.025)), suffix = "cm")] + #[schema(gui(slider(min = 0.0, max = 2.5, step = 0.025)), suffix = "cm")] pub pinch_trigger_distance: f32, #[schema(flag = "real-time")] From 0565e17b9b72ad7acbddf6055d8ea0040640475d Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:23:47 +0100 Subject: [PATCH 14/47] improve help strings for curl and pinch values --- alvr/session/src/settings.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 5f5e7cbc20..15c91c6f52 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -722,28 +722,28 @@ pub struct HandTrackingConfig { pub struct HandGestureConfig { #[schema(flag = "real-time")] #[schema(strings( - help = "How close the tips of your fingers need to be to register a pinch." + help = "How close the tips of your fingers need to be to register a pinch click." ))] #[schema(gui(slider(min = 0.0, max = 1.0, step = 0.01)), suffix = "cm")] pub pinch_touch_distance: f32, #[schema(flag = "real-time")] #[schema(strings( - help = "How close the tips of your fingers need to be to start registering a trigger pull." + help = "How close together the tips of your fingers need to be to start registering a pinch trigger pull." ))] #[schema(gui(slider(min = 0.0, max = 2.5, step = 0.025)), suffix = "cm")] pub pinch_trigger_distance: f32, #[schema(flag = "real-time")] #[schema(strings( - help = "How curled your fingers need to be to register a click." + help = "How close to your palm the tips of your fingers need to be to register a curl click." ))] #[schema(gui(slider(min = 0.0, max = 5.0)), suffix = "cm")] pub curl_touch_distance: f32, #[schema(flag = "real-time")] #[schema(strings( - help = "How curled your fingers need to be to start registering a trigger pull." + help = "How close to your palm the tips of your fingers need to be to start registering a trigger pull." ))] #[schema(gui(slider(min = 0.0, max = 10.0)), suffix = "cm")] pub curl_trigger_distance: f32, From b5171fca9487d1460fc758c285ae0047b67613fa Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:24:40 +0100 Subject: [PATCH 15/47] remove warns for curling --- alvr/server/src/tracking.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 5cc00a3fc5..a7a571242f 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -1,7 +1,7 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ glam::{EulerRot, Quat, Vec3}, - warn, DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, + DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, @@ -386,7 +386,6 @@ pub fn hands_to_gestures( - thumb_rad) / (curl_max + palm_depth + thumb_rad)) .clamp(0.0, 1.0); - warn!("thumb_curl: {}", thumb_curl); let index_pinch = thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; @@ -408,7 +407,6 @@ pub fn hands_to_gestures( - index_rad) / (curl_max + palm_depth + index_rad)) .clamp(0.0, 1.0); - warn!("index_curl: {}", index_curl); let middle_pinch = thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; @@ -430,7 +428,6 @@ pub fn hands_to_gestures( - middle_rad) / (curl_max + palm_depth + middle_rad)) .clamp(0.0, 1.0); - warn!("middle_curl: {}", middle_curl); let ring_pinch = thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; @@ -452,7 +449,6 @@ pub fn hands_to_gestures( - ring_rad) / (curl_max + palm_depth + ring_rad)) .clamp(0.0, 1.0); - warn!("ring_curl: {}", ring_curl); let little_pinch = thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; @@ -474,7 +470,6 @@ pub fn hands_to_gestures( - little_rad) / (curl_max + palm_depth + little_rad)) .clamp(0.0, 1.0); - warn!("little_curl: {}", little_curl); let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; From 4edb85935fb06c7ba8ac39fa2d32e97116ee9272 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:41:33 +0100 Subject: [PATCH 16/47] fix broken right hand binds --- alvr/server/src/tracking.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index a7a571242f..8128228ed1 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -5,7 +5,7 @@ use alvr_common::{ LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, - Y_CLICK_ID, + Y_CLICK_ID, Y_TOUCH_ID, B_TOUCH_ID, X_TOUCH_ID, A_TOUCH_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, @@ -496,13 +496,9 @@ pub fn hands_to_gestures( touch_bind: if device_id == *LEFT_HAND_ID { *Y_CLICK_ID } else { - 0 - }, - hover_bind: if device_id == *LEFT_HAND_ID { *B_CLICK_ID - } else { - 0 }, + hover_bind: 0, }, HandGesture { active: true, @@ -511,13 +507,9 @@ pub fn hands_to_gestures( touch_bind: if device_id == *LEFT_HAND_ID { *X_CLICK_ID } else { - 0 - }, - hover_bind: if device_id == *LEFT_HAND_ID { *A_CLICK_ID - } else { - 0 }, + hover_bind: 0, }, HandGesture { active: true, @@ -528,7 +520,7 @@ pub fn hands_to_gestures( } else { 0 }, - hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, + hover_bind: 0, }, HandGesture { active: true, From 92c4008972ed5826932b827062ac5da24703a24a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:02:08 +0100 Subject: [PATCH 17/47] remove button press workaround --- alvr/server/src/connection.rs | 84 +++++++++++++---------------------- 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 096b923e22..32ec8c8cd3 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -750,62 +750,42 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { // Handle gesture buttons { - let mut did_disable_skeleton = false; - let mut press_gesture_buttons = - |gestures: [HandGesture; 6]| { - for g in gestures { - if g.active { - // Workaround for gestures not triggering button presses - if !did_disable_skeleton { - unsafe { - crate::SetTracking( - tracking.target_timestamp.as_nanos() as _, - stats.tracker_pose_time_offset().as_secs_f32(), - ffi_motions.as_ptr(), - ffi_motions.len() as _, - ptr::null(), - ptr::null(), - track_controllers, - ) - }; - did_disable_skeleton = true; + let press_gesture_buttons = |gestures: [HandGesture; 6]| { + for g in gestures { + if g.active { + // Handle touch bind + if g.touch_bind != 0 { + unsafe { + crate::SetButton( + g.touch_bind, + crate::FfiButtonValue { + type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, + __bindgen_anon_1: + crate::FfiButtonValue__bindgen_ty_1 { + binary: g.touching.into(), + }, + }, + ) } - - // Handle touch bind - if g.touch_bind != 0 { - unsafe { - crate::SetButton( - g.touch_bind, - crate::FfiButtonValue { - type_: - crate::FfiButtonType_BUTTON_TYPE_BINARY, - __bindgen_anon_1: - crate::FfiButtonValue__bindgen_ty_1 { - binary: g.touching.into(), - }, - }, - ) - } - } - // Handle hover bind - if g.hover_bind != 0 { - unsafe { - crate::SetButton( - g.hover_bind, - crate::FfiButtonValue { - type_: - crate::FfiButtonType_BUTTON_TYPE_SCALAR, - __bindgen_anon_1: - crate::FfiButtonValue__bindgen_ty_1 { - scalar: g.hover_val.into(), - }, - }, - ) - } + } + // Handle hover bind + if g.hover_bind != 0 { + unsafe { + crate::SetButton( + g.hover_bind, + crate::FfiButtonValue { + type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, + __bindgen_anon_1: + crate::FfiButtonValue__bindgen_ty_1 { + scalar: g.hover_val.into(), + }, + }, + ) } } } - }; + } + }; let hand_gestures = [ tracking.hand_skeletons[0] From 61e7da2a0ed63dfaab20a77eea9e82b3826c4b41 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:03:56 +0100 Subject: [PATCH 18/47] support vertical joystick control using thumb curl all fingers and touch thumb to index finger. centre is between tip and intermediate bone --- alvr/server/src/connection.rs | 2 +- alvr/server/src/tracking.rs | 85 ++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 32ec8c8cd3..cf056925e6 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -750,7 +750,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { // Handle gesture buttons { - let press_gesture_buttons = |gestures: [HandGesture; 6]| { + let press_gesture_buttons = |gestures: [HandGesture; 8]| { for g in gestures { if g.active { // Handle touch bind diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 8128228ed1..07c0a03806 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -1,11 +1,12 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ - glam::{EulerRot, Quat, Vec3}, - DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, - LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, + glam::{EulerRot, Quat, Vec2, Vec3}, + warn, DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, + LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, + LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, - RIGHT_THUMBSTICK_CLICK_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, - Y_CLICK_ID, Y_TOUCH_ID, B_TOUCH_ID, X_TOUCH_ID, A_TOUCH_ID, + RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, + RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, Y_CLICK_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, @@ -342,7 +343,7 @@ pub fn hands_to_gestures( config: &HeadsetConfig, device_id: u64, hand_skeleton: [Pose; 26], -) -> [HandGesture; 6] { +) -> [HandGesture; 8] { if let Switch::Enabled(controllers) = &config.controllers { if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { if let Switch::Enabled(use_gestures) = &hand_tracking.use_gestures { @@ -365,9 +366,11 @@ pub fn hands_to_gestures( let curl_max = use_gestures.curl_trigger_distance * 0.01; let palm: Pose = gj[0]; + let thumb_proximal: Pose = gj[3]; let thumb_tip: Pose = gj[5]; let index_metacarpal: Pose = gj[6]; let index_proximal: Pose = gj[7]; + let index_intermediate: Pose = gj[8]; let index_tip: Pose = gj[10]; let middle_metacarpal: Pose = gj[11]; let middle_proximal: Pose = gj[12]; @@ -473,6 +476,24 @@ pub fn hands_to_gestures( let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; + let joystick_range = 0.005; + let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); + + let joystick_pos = Vec2 { + x: 0.0, + y: ((thumb_tip.position.distance(thumb_proximal.position) + - joystick_center.distance(thumb_proximal.position)) + / joystick_range) + .clamp(-1.0, 1.0), + }; + let joystick_contact = index_curl >= 0.75 + && grip_curl > 0.5 + && joystick_center.distance(thumb_tip.position) + <= joystick_range * 5.0; + + warn!("joystick contact: {}", joystick_contact); + warn!("joystick position: {}, {}", joystick_pos.x, joystick_pos.y); + return [ HandGesture { active: true, @@ -548,6 +569,44 @@ pub fn hands_to_gestures( }, hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, }, + HandGesture { + active: true, + touching: joystick_contact, + hover_val: if joystick_contact { + joystick_pos.x + } else { + 0.0 + }, + touch_bind: if device_id == *LEFT_HAND_ID { + *LEFT_THUMBSTICK_TOUCH_ID + } else { + *RIGHT_THUMBSTICK_TOUCH_ID + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *LEFT_THUMBSTICK_X_ID + } else { + *RIGHT_THUMBSTICK_X_ID + }, + }, + HandGesture { + active: true, + touching: joystick_contact, + hover_val: if joystick_contact { + joystick_pos.y + } else { + 0.0 + }, + touch_bind: if device_id == *LEFT_HAND_ID { + *LEFT_THUMBSTICK_TOUCH_ID + } else { + *RIGHT_THUMBSTICK_TOUCH_ID + }, + hover_bind: if device_id == *LEFT_HAND_ID { + *LEFT_THUMBSTICK_Y_ID + } else { + *RIGHT_THUMBSTICK_Y_ID + }, + }, ]; } } @@ -596,6 +655,20 @@ pub fn hands_to_gestures( touch_bind: 0, hover_bind: 0, }, + HandGesture { + active: false, + touching: false, + hover_val: 0.0, + touch_bind: 0, + hover_bind: 0, + }, + HandGesture { + active: false, + touching: false, + hover_val: 0.0, + touch_bind: 0, + hover_bind: 0, + }, ] } From fa01816e0927a8d7b91fd28b84ff824dd5c77574 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:21:16 +0100 Subject: [PATCH 19/47] disable index pinching when controlling the joystick --- alvr/server/src/tracking.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 07c0a03806..8c7548a1dc 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -497,8 +497,8 @@ pub fn hands_to_gestures( return [ HandGesture { active: true, - touching: index_pinch, - hover_val: index_trigger, + touching: index_pinch && !joystick_contact, + hover_val: if joystick_contact { 0 } else { index_trigger }, touch_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_CLICK_ID } else { From 580c404ace42cfe85c94e6716d5a296bfc78e65a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:22:38 +0100 Subject: [PATCH 20/47] 0.0 not 0 :c --- alvr/server/src/tracking.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 8c7548a1dc..919ff546b8 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -498,7 +498,7 @@ pub fn hands_to_gestures( HandGesture { active: true, touching: index_pinch && !joystick_contact, - hover_val: if joystick_contact { 0 } else { index_trigger }, + hover_val: if joystick_contact { 0.0 } else { index_trigger }, touch_bind: if device_id == *LEFT_HAND_ID { *LEFT_TRIGGER_CLICK_ID } else { From 20d6dbf54b9ffd74f16a3f0f5d3c05683e543b10 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 19:34:33 +0100 Subject: [PATCH 21/47] dual axis joystick support somewhat janky, likely just an issue with specific offsets and numbers --- alvr/server/src/tracking.rs | 39 +++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 919ff546b8..73baf94de1 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -374,6 +374,7 @@ pub fn hands_to_gestures( let index_tip: Pose = gj[10]; let middle_metacarpal: Pose = gj[11]; let middle_proximal: Pose = gj[12]; + let middle_intermediate: Pose = gj[13]; let middle_tip: Pose = gj[15]; let ring_metacarpal: Pose = gj[16]; let ring_proximal: Pose = gj[17]; @@ -479,20 +480,42 @@ pub fn hands_to_gestures( let joystick_range = 0.005; let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); + let joystick_up = (joystick_center + - middle_intermediate.position.lerp(middle_tip.position, 0.25)) + .normalize() + * joystick_range; + + let joystick_vertical_vec = + (joystick_center - thumb_proximal.position).normalize() * joystick_range; + let joystick_horizontal_vec = + joystick_vertical_vec.cross(joystick_up).normalize() * joystick_range; + + let joystick_vertical = (thumb_tip.position - joystick_center + + joystick_vertical_vec / 2.0) + .dot(joystick_vertical_vec) + / joystick_vertical_vec.length(); + let joystick_horizontal = + (thumb_tip.position - joystick_center - joystick_horizontal_vec / 2.0) + .dot(joystick_horizontal_vec) + / joystick_horizontal_vec.length(); + let joystick_pos = Vec2 { - x: 0.0, - y: ((thumb_tip.position.distance(thumb_proximal.position) - - joystick_center.distance(thumb_proximal.position)) - / joystick_range) - .clamp(-1.0, 1.0), + x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), + y: (joystick_vertical / joystick_range).clamp(-1.0, 1.0), }; let joystick_contact = index_curl >= 0.75 && grip_curl > 0.5 - && joystick_center.distance(thumb_tip.position) - <= joystick_range * 5.0; + && joystick_center.distance(thumb_tip.position) <= joystick_range * 5.0 + && (thumb_tip.position - joystick_center).dot(joystick_up) + / joystick_up.length() + <= joystick_range * 3.0; warn!("joystick contact: {}", joystick_contact); - warn!("joystick position: {}, {}", joystick_pos.x, joystick_pos.y); + warn!( + "joystick position: {}, {}", + joystick_horizontal, joystick_vertical + ); + warn!("joystick value: {}, {}", joystick_pos.x, joystick_pos.y); return [ HandGesture { From 15dc07e773a450aaf7e6edb02eafe83cbc089305 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 19:58:01 +0100 Subject: [PATCH 22/47] add gesture joystick deadzone --- alvr/server/src/tracking.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 73baf94de1..8ecb131ad9 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -477,7 +477,7 @@ pub fn hands_to_gestures( let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; - let joystick_range = 0.005; + let joystick_range = 0.01; let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); let joystick_up = (joystick_center @@ -494,10 +494,9 @@ pub fn hands_to_gestures( + joystick_vertical_vec / 2.0) .dot(joystick_vertical_vec) / joystick_vertical_vec.length(); - let joystick_horizontal = - (thumb_tip.position - joystick_center - joystick_horizontal_vec / 2.0) - .dot(joystick_horizontal_vec) - / joystick_horizontal_vec.length(); + let joystick_horizontal = (thumb_tip.position - joystick_center) + .dot(joystick_horizontal_vec) + / joystick_horizontal_vec.length(); let joystick_pos = Vec2 { x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), @@ -517,6 +516,8 @@ pub fn hands_to_gestures( ); warn!("joystick value: {}, {}", joystick_pos.x, joystick_pos.y); + let joystick_deadzone = 0.25; + return [ HandGesture { active: true, @@ -595,7 +596,7 @@ pub fn hands_to_gestures( HandGesture { active: true, touching: joystick_contact, - hover_val: if joystick_contact { + hover_val: if joystick_contact && joystick_pos.x >= joystick_deadzone { joystick_pos.x } else { 0.0 @@ -614,7 +615,7 @@ pub fn hands_to_gestures( HandGesture { active: true, touching: joystick_contact, - hover_val: if joystick_contact { + hover_val: if joystick_contact && joystick_pos.y >= joystick_deadzone { joystick_pos.y } else { 0.0 From ed5b26dd1c018d9b58dabafd2a97500464c4592d Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 20:03:08 +0100 Subject: [PATCH 23/47] config joystick deadzone and offset --- alvr/server/src/tracking.rs | 6 +++--- alvr/session/src/settings.rs | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 8ecb131ad9..5850baa5cc 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -493,10 +493,10 @@ pub fn hands_to_gestures( let joystick_vertical = (thumb_tip.position - joystick_center + joystick_vertical_vec / 2.0) .dot(joystick_vertical_vec) - / joystick_vertical_vec.length(); + / joystick_vertical_vec.length() + use_gestures.joystick_offset_vertical * 0.01; let joystick_horizontal = (thumb_tip.position - joystick_center) .dot(joystick_horizontal_vec) - / joystick_horizontal_vec.length(); + / joystick_horizontal_vec.length() + use_gestures.joystick_offset_horizontal * 0.01; let joystick_pos = Vec2 { x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), @@ -516,7 +516,7 @@ pub fn hands_to_gestures( ); warn!("joystick value: {}, {}", joystick_pos.x, joystick_pos.y); - let joystick_deadzone = 0.25; + let joystick_deadzone: f32 = use_gestures.joystick_deadzone * 0.01; return [ HandGesture { diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 15c91c6f52..219a500eb5 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -747,6 +747,18 @@ pub struct HandGestureConfig { ))] #[schema(gui(slider(min = 0.0, max = 10.0)), suffix = "cm")] pub curl_trigger_distance: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = 0.0, max = 100.0)), suffix = "%")] + pub joystick_deadzone: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")] + pub joystick_offset_horizontal: f32, + + #[schema(flag = "real-time")] + #[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")] + pub joystick_offset_vertical: f32, } #[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy)] @@ -1293,6 +1305,9 @@ pub fn session_settings_default() -> SettingsDefault { pinch_trigger_distance: 0.25, curl_touch_distance: 2.0, curl_trigger_distance: 2.5, + joystick_deadzone: 25.0, + joystick_offset_horizontal: 0.0, + joystick_offset_vertical: 0.0, }, }, enable_skeleton: true, From e1eaabe90473e5d9c3e27b1e1adab47c1e229673 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Thu, 17 Aug 2023 20:40:48 +0100 Subject: [PATCH 24/47] fix clippy errors --- alvr/server/src/connection.rs | 6 +++--- alvr/server/src/tracking.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index cf056925e6..c75228f840 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -777,7 +777,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - scalar: g.hover_val.into(), + scalar: g.hover_val, }, }, ) @@ -794,8 +794,8 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .map(|s| tracking::hands_to_gestures(config, *RIGHT_HAND_ID, s)), ]; - hand_gestures[0].map(|g| press_gesture_buttons(g)); - hand_gestures[1].map(|g| press_gesture_buttons(g)); + hand_gestures[0].map(press_gesture_buttons); + hand_gestures[1].map(press_gesture_buttons); } let mut hand_skeletons_enabled = false; diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 5850baa5cc..1dab72cca1 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -591,7 +591,7 @@ pub fn hands_to_gestures( } else { *RIGHT_THUMBSTICK_CLICK_ID }, - hover_bind: if device_id == *LEFT_HAND_ID { 0 } else { 0 }, + hover_bind: 0, }, HandGesture { active: true, From 4b2551efa4eb6b0ebd7cb1ab06c82145733a4589 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:14:31 +0100 Subject: [PATCH 25/47] format --- alvr/server/src/tracking.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 1dab72cca1..8b0f264e0e 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -493,10 +493,12 @@ pub fn hands_to_gestures( let joystick_vertical = (thumb_tip.position - joystick_center + joystick_vertical_vec / 2.0) .dot(joystick_vertical_vec) - / joystick_vertical_vec.length() + use_gestures.joystick_offset_vertical * 0.01; + / joystick_vertical_vec.length() + + use_gestures.joystick_offset_vertical * 0.01; let joystick_horizontal = (thumb_tip.position - joystick_center) .dot(joystick_horizontal_vec) - / joystick_horizontal_vec.length() + use_gestures.joystick_offset_horizontal * 0.01; + / joystick_horizontal_vec.length() + + use_gestures.joystick_offset_horizontal * 0.01; let joystick_pos = Vec2 { x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), From 716b8ebe0dbdd587ee1839bc5a4f0ce68275793c Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Fri, 25 Aug 2023 17:36:16 +0100 Subject: [PATCH 26/47] initial refactor work is_gesture_active --- alvr/server/src/hand_gestures.rs | 257 +++++++++++++++++++++++++++++++ alvr/server/src/lib.rs | 1 + alvr/server/src/tracking.rs | 2 + 3 files changed, 260 insertions(+) create mode 100644 alvr/server/src/hand_gestures.rs diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs new file mode 100644 index 0000000000..f90a4fc8b9 --- /dev/null +++ b/alvr/server/src/hand_gestures.rs @@ -0,0 +1,257 @@ +use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, hash::Hash, +}; + +use alvr_common::{anyhow::Result, Pose, glam::Vec2}; +use alvr_session::HandGestureConfig; + +#[derive(Debug, Copy, Clone)] +pub struct HandGesture { + pub touching: bool, + pub hover_val: f32, + pub touch_bind: u64, + pub hover_bind: u64, +} + +pub struct GestureAction { + last_activated: u128, + last_deactivated: u128, + active: bool, +} + +#[derive(Eq, Hash, PartialEq)] +enum HandGestureId {} + +pub struct HandGestureManager { + config: HandGestureConfig, + gesture_data: HashMap, +} + +impl HandGestureManager { + fn new(config: HandGestureConfig) -> Result { + Ok(Self { + config, + gesture_data: HashMap::new() + }) + } + + pub fn get_active_gestures(&self, hand_skeleton: [Pose; 26]) -> Vec { + // global joints + let gj = hand_skeleton; + + // if we model the tip of the finger as a spherical object, we should account for its radius + // these are intentionally under the average by ~5mm since the touch and trigger distances are already configurable in settings + let thumb_rad: f32 = 0.0075; // average thumb is ~20mm in diameter + let index_rad: f32 = 0.0065; // average index finger is ~18mm in diameter + let middle_rad: f32 = 0.0065; // average middle finger is ~18mm in diameter + let ring_rad: f32 = 0.006; // average ring finger is ~17mm in diameter + let little_rad: f32 = 0.005; // average pinky finger is ~15mm in diameter + let palm_depth: f32 = 0.005; // average palm bones are ~10mm from the skin + + // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers + let pinch_min = self.config.pinch_touch_distance * 0.01; + let pinch_max = self.config.pinch_trigger_distance * 0.01; + let curl_min = self.config.curl_touch_distance * 0.01; + let curl_max = self.config.curl_trigger_distance * 0.01; + + let palm: Pose = gj[0]; + let thumb_proximal: Pose = gj[3]; + let thumb_tip: Pose = gj[5]; + let index_metacarpal: Pose = gj[6]; + let index_proximal: Pose = gj[7]; + let index_intermediate: Pose = gj[8]; + let index_tip: Pose = gj[10]; + let middle_metacarpal: Pose = gj[11]; + let middle_proximal: Pose = gj[12]; + let middle_intermediate: Pose = gj[13]; + let middle_tip: Pose = gj[15]; + let ring_metacarpal: Pose = gj[16]; + let ring_proximal: Pose = gj[17]; + let ring_tip: Pose = gj[20]; + let little_metacarpal: Pose = gj[21]; + let little_proximal: Pose = gj[22]; + let little_tip: Pose = gj[25]; + + let thumb_curl = (1.0 + - (palm.position.distance(thumb_tip.position) - curl_min - palm_depth - thumb_rad) + / (curl_max + palm_depth + thumb_rad)) + .clamp(0.0, 1.0); + + let index_pinch = + thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; + let index_trigger = (1.0 + - (thumb_tip.position.distance(index_tip.position) + - pinch_min + - thumb_rad + - index_rad) + / (pinch_max + thumb_rad + index_rad)) + .clamp(0.0, 1.0); + + let index_curl = (1.0 + - (index_metacarpal + .position + .lerp(index_proximal.position, 0.5) + .distance(index_tip.position) + - curl_min + - palm_depth + - index_rad) + / (curl_max + palm_depth + index_rad)) + .clamp(0.0, 1.0); + + let middle_pinch = + thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; + let middle_trigger = (1.0 + - (thumb_tip.position.distance(middle_tip.position) + - pinch_min + - thumb_rad + - middle_rad) + / (pinch_max + thumb_rad + middle_rad)) + .clamp(0.0, 1.0); + + let middle_curl = (1.0 + - (middle_metacarpal + .position + .lerp(middle_proximal.position, 0.5) + .distance(middle_tip.position) + - curl_min + - palm_depth + - middle_rad) + / (curl_max + palm_depth + middle_rad)) + .clamp(0.0, 1.0); + + let ring_pinch = + thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; + let ring_trigger = (1.0 + - (thumb_tip.position.distance(ring_tip.position) - pinch_min - thumb_rad - ring_rad) + / (pinch_max + thumb_rad + ring_rad)) + .clamp(0.0, 1.0); + + let ring_curl = (1.0 + - (ring_metacarpal + .position + .lerp(ring_proximal.position, 0.5) + .distance(ring_tip.position) + - curl_min + - palm_depth + - ring_rad) + / (curl_max + palm_depth + ring_rad)) + .clamp(0.0, 1.0); + + let little_pinch = + thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; + let little_trigger = (1.0 + - (thumb_tip.position.distance(little_tip.position) + - pinch_min + - thumb_rad + - little_rad) + / (pinch_max + thumb_rad + little_rad)) + .clamp(0.0, 1.0); + + let little_curl = (1.0 + - (little_metacarpal + .position + .lerp(little_proximal.position, 0.5) + .distance(little_tip.position) + - curl_min + - palm_depth + - little_rad) + / (curl_max + palm_depth + little_rad)) + .clamp(0.0, 1.0); + + let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; + + let joystick_range = 0.01; + let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); + + let joystick_up = (joystick_center + - middle_intermediate.position.lerp(middle_tip.position, 0.25)) + .normalize() + * joystick_range; + + let joystick_vertical_vec = + (joystick_center - thumb_proximal.position).normalize() * joystick_range; + let joystick_horizontal_vec = + joystick_vertical_vec.cross(joystick_up).normalize() * joystick_range; + + let joystick_vertical = (thumb_tip.position - joystick_center + + joystick_vertical_vec / 2.0) + .dot(joystick_vertical_vec) + / joystick_vertical_vec.length() + + self.config.joystick_offset_vertical * 0.01; + let joystick_horizontal = (thumb_tip.position - joystick_center) + .dot(joystick_horizontal_vec) + / joystick_horizontal_vec.length() + + self.config.joystick_offset_horizontal * 0.01; + + let joystick_pos = Vec2 { + x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), + y: (joystick_vertical / joystick_range).clamp(-1.0, 1.0), + }; + let joystick_contact = index_curl >= 0.75 + && grip_curl > 0.5 + && joystick_center.distance(thumb_tip.position) <= joystick_range * 5.0 + && (thumb_tip.position - joystick_center).dot(joystick_up) / joystick_up.length() + <= joystick_range * 3.0; + + let joystick_deadzone: f32 = self.config.joystick_deadzone * 0.01; + + return vec![]; + } + + fn is_gesture_active( + &self, + gesture_id: HandGestureId, + first_anchor: Pose, + first_rad: f32, + second_anchor: Pose, + second_rad: f32, + halo: f32, + in_delay: u128, + out_delay: u128, + ) -> bool { + let is_active = first_anchor.position.distance(second_anchor.position) + < (halo + first_rad + second_rad); + + if !self.gesture_data.contains_key(&gesture_id) { + self.gesture_data.insert(gesture_id, GestureAction { + last_activated: 0, + last_deactivated: 0, + active: false, + }); + } + let g = self.gesture_data.get(&gesture_id).unwrap(); + + // If was already active, maintain state + if is_active && g.active { + return true; + } + // If was already inactive, maintain state + if !is_active && !g.active { + return false; + } + + // Get current time, for comparison + let time_millis = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); + + // If is becoming active, do not transition state unless the in_delay has passed since last deactivation + if is_active && !g.active { + if g.last_deactivated < time_millis - in_delay { + g.active = true; + return true; + } else { + return false; + } + } + + // If is becoming inactive, do not transition state unless the out_delay has passed since last activation + if !is_active && g.active { + if g.last_activated < time_millis - out_delay { + g.active = false; + return false; + } else { + return true; + } + } + } +} diff --git a/alvr/server/src/lib.rs b/alvr/server/src/lib.rs index 6a86516d1a..bc3f00475d 100644 --- a/alvr/server/src/lib.rs +++ b/alvr/server/src/lib.rs @@ -1,6 +1,7 @@ mod bitrate; mod connection; mod face_tracking; +mod hand_gestures; mod haptics; mod input_mapping; mod logging_backend; diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 8b0f264e0e..957523211f 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -16,6 +16,8 @@ use std::{ f32::consts::{FRAC_PI_2, PI}, }; +use crate::hand_gestures::HandGestureManager; + const DEG_TO_RAD: f32 = PI / 180.0; fn get_hand_skeleton_offsets(config: &HeadsetConfig) -> (Pose, Pose) { From 27e60dca7104f1bb3727ae67e6e4a1ea837121fb Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sat, 26 Aug 2023 01:54:16 +0100 Subject: [PATCH 27/47] migration to handgesturemanager (untested) --- alvr/server/src/connection.rs | 71 ++-- alvr/server/src/hand_gestures.rs | 550 ++++++++++++++++++++++--------- alvr/server/src/tracking.rs | 380 +-------------------- alvr/session/src/settings.rs | 24 ++ 4 files changed, 451 insertions(+), 574 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index c75228f840..140921b11a 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -5,7 +5,8 @@ use crate::{ input_mapping::ButtonMappingManager, sockets::WelcomeSocket, statistics::StatisticsManager, - tracking::{self, HandGesture, TrackingManager}, + tracking::{self, TrackingManager}, + hand_gestures::{HandGestureManager, trigger_hand_gesture_actions}, FfiFov, FfiViewsConfig, VideoPacket, BITRATE_MANAGER, DECODER_CONFIG, SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_MIRROR_SENDER, VIDEO_RECORDING_FILE, }; @@ -643,9 +644,11 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { }; let tracking_manager = Arc::new(Mutex::new(TrackingManager::new())); + let hand_gesture_manager = Arc::new(Mutex::new(HandGestureManager::new())); let tracking_receive_thread = thread::spawn({ let tracking_manager = Arc::clone(&tracking_manager); + let hand_gesture_manager = Arc::clone(&hand_gesture_manager); move || { let mut face_tracking_sink = settings @@ -657,7 +660,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { }); let mut track_controllers = 0u32; - if let Switch::Enabled(config) = settings.headset.controllers { + if let Switch::Enabled(config) = &settings.headset.controllers { track_controllers = config.tracked.into(); } @@ -748,54 +751,24 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { if let Some(stats) = &mut *STATISTICS_MANAGER.lock() { stats.report_tracking_received(tracking.target_timestamp); - // Handle gesture buttons - { - let press_gesture_buttons = |gestures: [HandGesture; 8]| { - for g in gestures { - if g.active { - // Handle touch bind - if g.touch_bind != 0 { - unsafe { - crate::SetButton( - g.touch_bind, - crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, - __bindgen_anon_1: - crate::FfiButtonValue__bindgen_ty_1 { - binary: g.touching.into(), - }, - }, - ) - } - } - // Handle hover bind - if g.hover_bind != 0 { - unsafe { - crate::SetButton( - g.hover_bind, - crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, - __bindgen_anon_1: - crate::FfiButtonValue__bindgen_ty_1 { - scalar: g.hover_val, - }, - }, - ) - } - } - } + // Handle hand gestures + if let Switch::Enabled(controllers_config) = &settings.headset.controllers { + if let Switch::Enabled(hand_tracking_config) = &controllers_config.hand_tracking { + if let Switch::Enabled(gestures_config) = &hand_tracking_config.use_gestures { + let mut hand_gesture_manager_lock = hand_gesture_manager.lock(); + + trigger_hand_gesture_actions( + *LEFT_HAND_ID, + hand_gesture_manager_lock.get_active_gestures(left_hand_skeleton.unwrap(), gestures_config.clone(), *LEFT_HAND_ID) + ); + trigger_hand_gesture_actions( + *RIGHT_HAND_ID, + hand_gesture_manager_lock.get_active_gestures(right_hand_skeleton.unwrap(), gestures_config.clone(), *RIGHT_HAND_ID) + ); + + drop(hand_gesture_manager_lock); } - }; - - let hand_gestures = [ - tracking.hand_skeletons[0] - .map(|s| tracking::hands_to_gestures(config, *LEFT_HAND_ID, s)), - tracking.hand_skeletons[1] - .map(|s| tracking::hands_to_gestures(config, *RIGHT_HAND_ID, s)), - ]; - - hand_gestures[0].map(press_gesture_buttons); - hand_gestures[1].map(press_gesture_buttons); + } } let mut hand_skeletons_enabled = false; diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index f90a4fc8b9..30b3c874e5 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -1,42 +1,93 @@ use std::{ collections::HashMap, - time::{SystemTime, UNIX_EPOCH}, hash::Hash, + hash::Hash, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use alvr_common::{ + glam::{Quat, Vec2, Vec3}, + Pose, A_CLICK_ID, B_CLICK_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, + LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, + LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, + RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, + RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, + X_CLICK_ID, Y_CLICK_ID, }; -use alvr_common::{anyhow::Result, Pose, glam::Vec2}; use alvr_session::HandGestureConfig; -#[derive(Debug, Copy, Clone)] +fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { + Pose { + orientation: Quat::from_xyzw( + a.orientation.x * (1.0 - fac) + (b.position.x * fac), + a.orientation.y * (1.0 - fac) + (b.orientation.y * fac), + a.orientation.z * (1.0 - fac) + (b.orientation.z * fac), + a.orientation.w * (1.0 - fac) + (b.orientation.w * fac), + ), + position: Vec3 { + x: a.position.x * (1.0 - fac) + (b.position.x * fac), + y: a.position.y * (1.0 - fac) + (b.position.y * fac), + z: a.position.z * (1.0 - fac) + (b.position.z * fac), + }, + } +} + +#[derive(Debug, Clone)] pub struct HandGesture { - pub touching: bool, - pub hover_val: f32, - pub touch_bind: u64, - pub hover_bind: u64, + pub id: HandGestureId, + pub active: bool, + pub hover: f32, } pub struct GestureAction { last_activated: u128, last_deactivated: u128, + entering: bool, + entering_since: u128, + exiting: bool, + exiting_since: u128, active: bool, } -#[derive(Eq, Hash, PartialEq)] -enum HandGestureId {} +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq)] +#[allow(dead_code)] +pub enum HandGestureId { + // Pinches + ThumbIndexPinch, + ThumbMiddlePinch, + ThumbRingPinch, + ThumbLittlePinch, + // Curls + ThumbCurl, + IndexCurl, + MiddleCurl, + RingCurl, + LittleCurl, + GripCurl, + // Complex + JoystickX, + JoystickY, +} pub struct HandGestureManager { - config: HandGestureConfig, - gesture_data: HashMap, + gesture_data_left: HashMap, + gesture_data_right: HashMap, } impl HandGestureManager { - fn new(config: HandGestureConfig) -> Result { - Ok(Self { - config, - gesture_data: HashMap::new() - }) + pub fn new() -> Self { + Self { + gesture_data_left: HashMap::new(), + gesture_data_right: HashMap::new(), + } } - pub fn get_active_gestures(&self, hand_skeleton: [Pose; 26]) -> Vec { + pub fn get_active_gestures( + &mut self, + hand_skeleton: [Pose; 26], + config: HandGestureConfig, + device_id: u64, + ) -> Vec { // global joints let gj = hand_skeleton; @@ -50,10 +101,10 @@ impl HandGestureManager { let palm_depth: f32 = 0.005; // average palm bones are ~10mm from the skin // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers - let pinch_min = self.config.pinch_touch_distance * 0.01; - let pinch_max = self.config.pinch_trigger_distance * 0.01; - let curl_min = self.config.curl_touch_distance * 0.01; - let curl_max = self.config.curl_trigger_distance * 0.01; + let pinch_min = config.pinch_touch_distance * 0.01; + let pinch_max = config.pinch_trigger_distance * 0.01; + let curl_min = config.curl_touch_distance * 0.01; + let curl_max = config.curl_trigger_distance * 0.01; let palm: Pose = gj[0]; let thumb_proximal: Pose = gj[3]; @@ -73,94 +124,135 @@ impl HandGestureManager { let little_proximal: Pose = gj[22]; let little_tip: Pose = gj[25]; - let thumb_curl = (1.0 - - (palm.position.distance(thumb_tip.position) - curl_min - palm_depth - thumb_rad) - / (curl_max + palm_depth + thumb_rad)) - .clamp(0.0, 1.0); - - let index_pinch = - thumb_tip.position.distance(index_tip.position) < pinch_min + thumb_rad + index_rad; - let index_trigger = (1.0 - - (thumb_tip.position.distance(index_tip.position) - - pinch_min - - thumb_rad - - index_rad) - / (pinch_max + thumb_rad + index_rad)) - .clamp(0.0, 1.0); - - let index_curl = (1.0 - - (index_metacarpal - .position - .lerp(index_proximal.position, 0.5) - .distance(index_tip.position) - - curl_min - - palm_depth - - index_rad) - / (curl_max + palm_depth + index_rad)) - .clamp(0.0, 1.0); - - let middle_pinch = - thumb_tip.position.distance(middle_tip.position) < pinch_min + thumb_rad + middle_rad; - let middle_trigger = (1.0 - - (thumb_tip.position.distance(middle_tip.position) - - pinch_min - - thumb_rad - - middle_rad) - / (pinch_max + thumb_rad + middle_rad)) - .clamp(0.0, 1.0); - - let middle_curl = (1.0 - - (middle_metacarpal - .position - .lerp(middle_proximal.position, 0.5) - .distance(middle_tip.position) - - curl_min - - palm_depth - - middle_rad) - / (curl_max + palm_depth + middle_rad)) - .clamp(0.0, 1.0); - - let ring_pinch = - thumb_tip.position.distance(ring_tip.position) < pinch_min + thumb_rad + ring_rad; - let ring_trigger = (1.0 - - (thumb_tip.position.distance(ring_tip.position) - pinch_min - thumb_rad - ring_rad) - / (pinch_max + thumb_rad + ring_rad)) - .clamp(0.0, 1.0); - - let ring_curl = (1.0 - - (ring_metacarpal - .position - .lerp(ring_proximal.position, 0.5) - .distance(ring_tip.position) - - curl_min - - palm_depth - - ring_rad) - / (curl_max + palm_depth + ring_rad)) - .clamp(0.0, 1.0); - - let little_pinch = - thumb_tip.position.distance(little_tip.position) < pinch_min + thumb_rad + little_rad; - let little_trigger = (1.0 - - (thumb_tip.position.distance(little_tip.position) - - pinch_min - - thumb_rad - - little_rad) - / (pinch_max + thumb_rad + little_rad)) - .clamp(0.0, 1.0); - - let little_curl = (1.0 - - (little_metacarpal - .position - .lerp(little_proximal.position, 0.5) - .distance(little_tip.position) - - curl_min - - palm_depth - - little_rad) - / (curl_max + palm_depth + little_rad)) - .clamp(0.0, 1.0); - + let mut gestures: Vec = Vec::new(); + + // Thumb & index pinch + gestures.push(HandGesture { + id: HandGestureId::ThumbIndexPinch, + active: self.is_gesture_active( + HandGestureId::ThumbIndexPinch, + thumb_tip, + thumb_rad, + index_tip, + index_rad, + pinch_max, + config.repeat_delay, + config.start_delay, + config.stop_delay, + device_id, + ), + hover: self.get_gesture_hover( + thumb_tip, thumb_rad, index_tip, index_rad, pinch_min, pinch_max, + ), + }); + + // Thumb & middle pinch + gestures.push(HandGesture { + id: HandGestureId::ThumbMiddlePinch, + active: self.is_gesture_active( + HandGestureId::ThumbMiddlePinch, + thumb_tip, + thumb_rad, + middle_tip, + middle_rad, + pinch_max, + config.repeat_delay, + config.start_delay, + config.stop_delay, + device_id, + ), + hover: self.get_gesture_hover( + thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min, pinch_max, + ), + }); + + // Thumb & ring pinch + gestures.push(HandGesture { + id: HandGestureId::ThumbRingPinch, + active: self.is_gesture_active( + HandGestureId::ThumbRingPinch, + thumb_tip, + thumb_rad, + ring_tip, + ring_rad, + pinch_max, + config.repeat_delay, + config.start_delay, + config.stop_delay, + device_id, + ), + hover: self.get_gesture_hover( + thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min, pinch_max, + ), + }); + + // Thumb & little pinch + gestures.push(HandGesture { + id: HandGestureId::ThumbLittlePinch, + active: self.is_gesture_active( + HandGestureId::ThumbLittlePinch, + thumb_tip, + thumb_rad, + little_tip, + little_rad, + pinch_max, + config.repeat_delay, + config.start_delay, + config.stop_delay, + device_id, + ), + hover: self.get_gesture_hover( + thumb_tip, thumb_rad, little_tip, little_rad, pinch_min, pinch_max, + ), + }); + + // Finger curls + let thumb_curl = + self.get_gesture_hover(palm, palm_depth, thumb_tip, thumb_rad, curl_min, curl_max); + let index_curl = self.get_gesture_hover( + lerp_pose(index_metacarpal, index_proximal, 0.5), + palm_depth, + index_tip, + index_rad, + curl_min, + curl_max, + ); + let middle_curl = self.get_gesture_hover( + lerp_pose(middle_metacarpal, middle_proximal, 0.5), + palm_depth, + middle_tip, + middle_rad, + curl_min, + curl_max, + ); + let ring_curl = self.get_gesture_hover( + lerp_pose(ring_metacarpal, ring_proximal, 0.5), + palm_depth, + ring_tip, + ring_rad, + curl_min, + curl_max, + ); + let little_curl = self.get_gesture_hover( + lerp_pose(little_metacarpal, little_proximal, 0.5), + palm_depth, + little_tip, + little_rad, + curl_min, + curl_max, + ); + + // Grip (NEEDS REFINING) let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; + let grip_active = grip_curl == 1.0; + + gestures.push(HandGesture { + id: HandGestureId::GripCurl, + active: grip_active, + hover: grip_curl, + }); + // Joystick (NEEDS REFINING) let joystick_range = 0.01; let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); @@ -178,11 +270,11 @@ impl HandGestureManager { + joystick_vertical_vec / 2.0) .dot(joystick_vertical_vec) / joystick_vertical_vec.length() - + self.config.joystick_offset_vertical * 0.01; + + config.joystick_offset_vertical * 0.01; let joystick_horizontal = (thumb_tip.position - joystick_center) .dot(joystick_horizontal_vec) / joystick_horizontal_vec.length() - + self.config.joystick_offset_horizontal * 0.01; + + config.joystick_offset_horizontal * 0.01; let joystick_pos = Vec2 { x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), @@ -194,63 +286,225 @@ impl HandGestureManager { && (thumb_tip.position - joystick_center).dot(joystick_up) / joystick_up.length() <= joystick_range * 3.0; - let joystick_deadzone: f32 = self.config.joystick_deadzone * 0.01; + let joystick_deadzone: f32 = config.joystick_deadzone * 0.01; + + gestures.push(HandGesture { + id: HandGestureId::ThumbCurl, + active: thumb_curl == 1.0, + hover: thumb_curl, + }); + gestures.push(HandGesture { + id: HandGestureId::JoystickX, + active: joystick_contact, + hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { + joystick_pos.x + } else { + 0.0 + }, + }); + gestures.push(HandGesture { + id: HandGestureId::JoystickY, + active: joystick_contact, + hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { + joystick_pos.y + } else { + 0.0 + }, + }); - return vec![]; + gestures } fn is_gesture_active( - &self, + &mut self, gesture_id: HandGestureId, first_anchor: Pose, - first_rad: f32, + first_radius: f32, second_anchor: Pose, - second_rad: f32, - halo: f32, - in_delay: u128, - out_delay: u128, + second_radius: f32, + activation_dist: f32, + repeat_delay: u32, + in_delay: u32, + out_delay: u32, + device_id: u64, ) -> bool { - let is_active = first_anchor.position.distance(second_anchor.position) - < (halo + first_rad + second_rad); - - if !self.gesture_data.contains_key(&gesture_id) { - self.gesture_data.insert(gesture_id, GestureAction { - last_activated: 0, - last_deactivated: 0, - active: false, - }); - } - let g = self.gesture_data.get(&gesture_id).unwrap(); + let in_range = first_anchor.position.distance(second_anchor.position) + < (activation_dist + first_radius + second_radius); - // If was already active, maintain state - if is_active && g.active { - return true; + let gesture_data = if device_id == *LEFT_HAND_ID { + &mut self.gesture_data_left + } else { + &mut self.gesture_data_right + }; + + if !gesture_data.contains_key(&gesture_id) { + gesture_data.insert( + gesture_id, + GestureAction { + last_activated: 0, + last_deactivated: 0, + entering: false, + entering_since: 0, + exiting: false, + exiting_since: 0, + active: false, + }, + ); } - // If was already inactive, maintain state - if !is_active && !g.active { - return false; + let g: &mut GestureAction = gesture_data.get_mut(&gesture_id).unwrap(); + + // Disable entering/exiting state if we leave/enter range + if in_range { + g.exiting = false; + } else { + g.entering = false; } + // Default to maintaining state + let mut new_active_state = g.active; + // Get current time, for comparison - let time_millis = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); + let time_millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_millis(0)) + .as_millis(); + + // Transitioning from inactive to active + if in_range && !g.active { + // Don't transition state unless the duration of repeat_delay has passed since last deactivation + if g.last_deactivated < time_millis - u128::from(repeat_delay) { + if g.entering { + // Don't transition state unless gesture has been in range for the duration of in_delay + if g.entering_since < time_millis - u128::from(in_delay) { + g.last_activated = time_millis; + g.entering = false; + new_active_state = true; + } + } else { + // Begin tracking entering state + g.entering = true; + g.entering_since = time_millis; + } + } + } - // If is becoming active, do not transition state unless the in_delay has passed since last deactivation - if is_active && !g.active { - if g.last_deactivated < time_millis - in_delay { - g.active = true; - return true; + // Transitioning from inactive to active + if !in_range && g.active { + if g.exiting { + // Don't transition state unless gesture has been out of range for the duration of out_delay + if g.exiting_since < time_millis - u128::from(out_delay) { + g.last_deactivated = time_millis; + g.exiting = false; + new_active_state = false; + } } else { - return false; + // Begin tracking exiting state + g.exiting = true; + g.exiting_since = time_millis; } } - // If is becoming inactive, do not transition state unless the out_delay has passed since last activation - if !is_active && g.active { - if g.last_activated < time_millis - out_delay { - g.active = false; - return false; - } else { - return true; + g.active = new_active_state; + + g.active + } + + fn get_gesture_hover( + &self, + first_anchor: Pose, + first_radius: f32, + second_anchor: Pose, + second_radius: f32, + min_dist: f32, + max_dist: f32, + ) -> f32 { + (1.0 - (first_anchor.position.distance(second_anchor.position) + - min_dist + - first_radius + - second_radius) + / (max_dist + first_radius + second_radius)) + .clamp(0.0, 1.0) + } +} + +fn get_active_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { + if device_id == *LEFT_HAND_ID { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_CLICK_ID), + HandGestureId::ThumbMiddlePinch => Some(*Y_CLICK_ID), + HandGestureId::ThumbRingPinch => Some(*X_CLICK_ID), + HandGestureId::ThumbLittlePinch => Some(*MENU_CLICK_ID), + HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_CLICK_ID), + HandGestureId::ThumbCurl => Some(*LEFT_THUMBSTICK_CLICK_ID), + HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_TOUCH_ID), + HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_TOUCH_ID), + _ => None, + } + } else { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_CLICK_ID), + HandGestureId::ThumbMiddlePinch => Some(*B_CLICK_ID), + HandGestureId::ThumbRingPinch => Some(*A_CLICK_ID), + HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_CLICK_ID), + HandGestureId::ThumbCurl => Some(*RIGHT_THUMBSTICK_CLICK_ID), + HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_TOUCH_ID), + HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_TOUCH_ID), + _ => None, + } + } +} + +fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { + if device_id == *LEFT_HAND_ID { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_VALUE_ID), + HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_VALUE_ID), + HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_X_ID), + HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_Y_ID), + _ => None, + } + } else { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_VALUE_ID), + HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_VALUE_ID), + HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_X_ID), + HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_Y_ID), + _ => None, + } + } +} + +pub fn trigger_hand_gesture_actions(device_id: u64, gestures: Vec) { + for gesture in gestures.iter() { + // Active bind + let active_bind = get_active_bind_for_gesture(device_id, gesture.id); + if active_bind.is_some() { + unsafe { + crate::SetButton( + active_bind.unwrap(), + crate::FfiButtonValue { + type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, + __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { + binary: gesture.active.into(), + }, + }, + ); + } + } + + // Hover bind + let hover_bind = get_hover_bind_for_gesture(device_id, gesture.id); + if hover_bind.is_some() { + unsafe { + crate::SetButton( + hover_bind.unwrap(), + crate::FfiButtonValue { + type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, + __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { + scalar: gesture.hover.into(), + }, + }, + ); } } } diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 957523211f..61606e9091 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -1,12 +1,7 @@ use crate::{to_ffi_quat, FfiDeviceMotion, FfiHandSkeleton}; use alvr_common::{ - glam::{EulerRot, Quat, Vec2, Vec3}, - warn, DeviceMotion, Pose, A_CLICK_ID, B_CLICK_ID, HEAD_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, - LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, - LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, - MENU_CLICK_ID, RIGHT_HAND_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, - RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, - RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, X_CLICK_ID, Y_CLICK_ID, + glam::{EulerRot, Quat, Vec3}, + DeviceMotion, Pose, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID, }; use alvr_session::{ settings_schema::Switch, HeadsetConfig, PositionRecenteringMode, RotationRecenteringMode, @@ -16,8 +11,6 @@ use std::{ f32::consts::{FRAC_PI_2, PI}, }; -use crate::hand_gestures::HandGestureManager; - const DEG_TO_RAD: f32 = PI / 180.0; fn get_hand_skeleton_offsets(config: &HeadsetConfig) -> (Pose, Pose) { @@ -72,6 +65,7 @@ impl TrackingManager { TrackingManager { last_head_pose: Pose::default(), recentering_origin: Pose::default(), + } } @@ -332,374 +326,6 @@ pub fn to_openvr_hand_skeleton( ] } -#[derive(Debug, Copy, Clone)] -pub struct HandGesture { - pub active: bool, - pub touching: bool, - pub hover_val: f32, - pub touch_bind: u64, - pub hover_bind: u64, -} - -pub fn hands_to_gestures( - config: &HeadsetConfig, - device_id: u64, - hand_skeleton: [Pose; 26], -) -> [HandGesture; 8] { - if let Switch::Enabled(controllers) = &config.controllers { - if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { - if let Switch::Enabled(use_gestures) = &hand_tracking.use_gestures { - // global joints - let gj = hand_skeleton; - - // if we model the tip of the finger as a spherical object, we should account for its radius - // these are intentionally under the average by ~5mm since the touch and trigger distances are already configurable in settings - let thumb_rad: f32 = 0.0075; // average thumb is ~20mm in diameter - let index_rad: f32 = 0.0065; // average index finger is ~18mm in diameter - let middle_rad: f32 = 0.0065; // average middle finger is ~18mm in diameter - let ring_rad: f32 = 0.006; // average ring finger is ~17mm in diameter - let little_rad: f32 = 0.005; // average pinky finger is ~15mm in diameter - let palm_depth: f32 = 0.005; // average palm bones are ~10mm from the skin - - // we add the radius of the finger and thumb because we're measuring the distance between the surface of them, not their centers - let pinch_min = use_gestures.pinch_touch_distance * 0.01; - let pinch_max = use_gestures.pinch_trigger_distance * 0.01; - let curl_min = use_gestures.curl_touch_distance * 0.01; - let curl_max = use_gestures.curl_trigger_distance * 0.01; - - let palm: Pose = gj[0]; - let thumb_proximal: Pose = gj[3]; - let thumb_tip: Pose = gj[5]; - let index_metacarpal: Pose = gj[6]; - let index_proximal: Pose = gj[7]; - let index_intermediate: Pose = gj[8]; - let index_tip: Pose = gj[10]; - let middle_metacarpal: Pose = gj[11]; - let middle_proximal: Pose = gj[12]; - let middle_intermediate: Pose = gj[13]; - let middle_tip: Pose = gj[15]; - let ring_metacarpal: Pose = gj[16]; - let ring_proximal: Pose = gj[17]; - let ring_tip: Pose = gj[20]; - let little_metacarpal: Pose = gj[21]; - let little_proximal: Pose = gj[22]; - let little_tip: Pose = gj[25]; - - let thumb_curl = (1.0 - - (palm.position.distance(thumb_tip.position) - - curl_min - - palm_depth - - thumb_rad) - / (curl_max + palm_depth + thumb_rad)) - .clamp(0.0, 1.0); - - let index_pinch = thumb_tip.position.distance(index_tip.position) - < pinch_min + thumb_rad + index_rad; - let index_trigger = (1.0 - - (thumb_tip.position.distance(index_tip.position) - - pinch_min - - thumb_rad - - index_rad) - / (pinch_max + thumb_rad + index_rad)) - .clamp(0.0, 1.0); - - let index_curl = (1.0 - - (index_metacarpal - .position - .lerp(index_proximal.position, 0.5) - .distance(index_tip.position) - - curl_min - - palm_depth - - index_rad) - / (curl_max + palm_depth + index_rad)) - .clamp(0.0, 1.0); - - let middle_pinch = thumb_tip.position.distance(middle_tip.position) - < pinch_min + thumb_rad + middle_rad; - let middle_trigger = (1.0 - - (thumb_tip.position.distance(middle_tip.position) - - pinch_min - - thumb_rad - - middle_rad) - / (pinch_max + thumb_rad + middle_rad)) - .clamp(0.0, 1.0); - - let middle_curl = (1.0 - - (middle_metacarpal - .position - .lerp(middle_proximal.position, 0.5) - .distance(middle_tip.position) - - curl_min - - palm_depth - - middle_rad) - / (curl_max + palm_depth + middle_rad)) - .clamp(0.0, 1.0); - - let ring_pinch = thumb_tip.position.distance(ring_tip.position) - < pinch_min + thumb_rad + ring_rad; - let ring_trigger = (1.0 - - (thumb_tip.position.distance(ring_tip.position) - - pinch_min - - thumb_rad - - ring_rad) - / (pinch_max + thumb_rad + ring_rad)) - .clamp(0.0, 1.0); - - let ring_curl = (1.0 - - (ring_metacarpal - .position - .lerp(ring_proximal.position, 0.5) - .distance(ring_tip.position) - - curl_min - - palm_depth - - ring_rad) - / (curl_max + palm_depth + ring_rad)) - .clamp(0.0, 1.0); - - let little_pinch = thumb_tip.position.distance(little_tip.position) - < pinch_min + thumb_rad + little_rad; - let little_trigger = (1.0 - - (thumb_tip.position.distance(little_tip.position) - - pinch_min - - thumb_rad - - little_rad) - / (pinch_max + thumb_rad + little_rad)) - .clamp(0.0, 1.0); - - let little_curl = (1.0 - - (little_metacarpal - .position - .lerp(little_proximal.position, 0.5) - .distance(little_tip.position) - - curl_min - - palm_depth - - little_rad) - / (curl_max + palm_depth + little_rad)) - .clamp(0.0, 1.0); - - let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; - - let joystick_range = 0.01; - let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); - - let joystick_up = (joystick_center - - middle_intermediate.position.lerp(middle_tip.position, 0.25)) - .normalize() - * joystick_range; - - let joystick_vertical_vec = - (joystick_center - thumb_proximal.position).normalize() * joystick_range; - let joystick_horizontal_vec = - joystick_vertical_vec.cross(joystick_up).normalize() * joystick_range; - - let joystick_vertical = (thumb_tip.position - joystick_center - + joystick_vertical_vec / 2.0) - .dot(joystick_vertical_vec) - / joystick_vertical_vec.length() - + use_gestures.joystick_offset_vertical * 0.01; - let joystick_horizontal = (thumb_tip.position - joystick_center) - .dot(joystick_horizontal_vec) - / joystick_horizontal_vec.length() - + use_gestures.joystick_offset_horizontal * 0.01; - - let joystick_pos = Vec2 { - x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), - y: (joystick_vertical / joystick_range).clamp(-1.0, 1.0), - }; - let joystick_contact = index_curl >= 0.75 - && grip_curl > 0.5 - && joystick_center.distance(thumb_tip.position) <= joystick_range * 5.0 - && (thumb_tip.position - joystick_center).dot(joystick_up) - / joystick_up.length() - <= joystick_range * 3.0; - - warn!("joystick contact: {}", joystick_contact); - warn!( - "joystick position: {}, {}", - joystick_horizontal, joystick_vertical - ); - warn!("joystick value: {}, {}", joystick_pos.x, joystick_pos.y); - - let joystick_deadzone: f32 = use_gestures.joystick_deadzone * 0.01; - - return [ - HandGesture { - active: true, - touching: index_pinch && !joystick_contact, - hover_val: if joystick_contact { 0.0 } else { index_trigger }, - touch_bind: if device_id == *LEFT_HAND_ID { - *LEFT_TRIGGER_CLICK_ID - } else { - *RIGHT_TRIGGER_CLICK_ID - }, - hover_bind: if device_id == *LEFT_HAND_ID { - *LEFT_TRIGGER_VALUE_ID - } else { - *RIGHT_TRIGGER_VALUE_ID - }, - }, - HandGesture { - active: true, - touching: middle_pinch, - hover_val: middle_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { - *Y_CLICK_ID - } else { - *B_CLICK_ID - }, - hover_bind: 0, - }, - HandGesture { - active: true, - touching: ring_pinch, - hover_val: ring_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { - *X_CLICK_ID - } else { - *A_CLICK_ID - }, - hover_bind: 0, - }, - HandGesture { - active: true, - touching: little_pinch, - hover_val: little_trigger, - touch_bind: if device_id == *LEFT_HAND_ID { - *MENU_CLICK_ID - } else { - 0 - }, - hover_bind: 0, - }, - HandGesture { - active: true, - touching: grip_curl == 1.0, - hover_val: grip_curl, - touch_bind: if device_id == *LEFT_HAND_ID { - *LEFT_SQUEEZE_CLICK_ID - } else { - *RIGHT_SQUEEZE_CLICK_ID - }, - hover_bind: if device_id == *LEFT_HAND_ID { - *LEFT_SQUEEZE_VALUE_ID - } else { - *RIGHT_SQUEEZE_VALUE_ID - }, - }, - HandGesture { - active: true, - touching: thumb_curl == 1.0, - hover_val: thumb_curl, - touch_bind: if device_id == *LEFT_HAND_ID { - *LEFT_THUMBSTICK_CLICK_ID - } else { - *RIGHT_THUMBSTICK_CLICK_ID - }, - hover_bind: 0, - }, - HandGesture { - active: true, - touching: joystick_contact, - hover_val: if joystick_contact && joystick_pos.x >= joystick_deadzone { - joystick_pos.x - } else { - 0.0 - }, - touch_bind: if device_id == *LEFT_HAND_ID { - *LEFT_THUMBSTICK_TOUCH_ID - } else { - *RIGHT_THUMBSTICK_TOUCH_ID - }, - hover_bind: if device_id == *LEFT_HAND_ID { - *LEFT_THUMBSTICK_X_ID - } else { - *RIGHT_THUMBSTICK_X_ID - }, - }, - HandGesture { - active: true, - touching: joystick_contact, - hover_val: if joystick_contact && joystick_pos.y >= joystick_deadzone { - joystick_pos.y - } else { - 0.0 - }, - touch_bind: if device_id == *LEFT_HAND_ID { - *LEFT_THUMBSTICK_TOUCH_ID - } else { - *RIGHT_THUMBSTICK_TOUCH_ID - }, - hover_bind: if device_id == *LEFT_HAND_ID { - *LEFT_THUMBSTICK_Y_ID - } else { - *RIGHT_THUMBSTICK_Y_ID - }, - }, - ]; - } - } - } - - [ - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - HandGesture { - active: false, - touching: false, - hover_val: 0.0, - touch_bind: 0, - hover_bind: 0, - }, - ] -} - pub fn to_ffi_motion(device_id: u64, motion: DeviceMotion) -> FfiDeviceMotion { FfiDeviceMotion { deviceID: device_id, diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 219a500eb5..7ef6592418 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -759,6 +759,27 @@ pub struct HandGestureConfig { #[schema(flag = "real-time")] #[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")] pub joystick_offset_vertical: f32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How long the gesture must be continuously held before it is activated." + ))] + #[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")] + pub start_delay: u32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How long the gesture must be continuously released before it is deactivated." + ))] + #[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")] + pub stop_delay: u32, + + #[schema(flag = "real-time")] + #[schema(strings( + help = "How long the after the gesture has been deactivated before it can be activated again." + ))] + #[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")] + pub repeat_delay: u32, } #[derive(SettingsSchema, Serialize, Deserialize, Clone, Copy)] @@ -1308,6 +1329,9 @@ pub fn session_settings_default() -> SettingsDefault { joystick_deadzone: 25.0, joystick_offset_horizontal: 0.0, joystick_offset_vertical: 0.0, + repeat_delay: 100, + start_delay: 50, + stop_delay: 50 }, }, enable_skeleton: true, From a57d7cfa69ebf8a8a50b51d125fe2f749825c45c Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 11:32:12 +0100 Subject: [PATCH 28/47] make gesture activation work --- alvr/server/src/connection.rs | 38 +++++++++++++++++++++++--------- alvr/server/src/hand_gestures.rs | 17 +++++--------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 140921b11a..c25f2f8c83 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -1,12 +1,12 @@ use crate::{ bitrate::BitrateManager, face_tracking::FaceTrackingSink, + hand_gestures::{trigger_hand_gesture_actions, HandGestureManager}, haptics, input_mapping::ButtonMappingManager, sockets::WelcomeSocket, statistics::StatisticsManager, tracking::{self, TrackingManager}, - hand_gestures::{HandGestureManager, trigger_hand_gesture_actions}, FfiFov, FfiViewsConfig, VideoPacket, BITRATE_MANAGER, DECODER_CONFIG, SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_MIRROR_SENDER, VIDEO_RECORDING_FILE, }; @@ -753,18 +753,34 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { // Handle hand gestures if let Switch::Enabled(controllers_config) = &settings.headset.controllers { - if let Switch::Enabled(hand_tracking_config) = &controllers_config.hand_tracking { - if let Switch::Enabled(gestures_config) = &hand_tracking_config.use_gestures { + if let Switch::Enabled(hand_tracking_config) = + &controllers_config.hand_tracking + { + if let Switch::Enabled(gestures_config) = + &hand_tracking_config.use_gestures + { let mut hand_gesture_manager_lock = hand_gesture_manager.lock(); - trigger_hand_gesture_actions( - *LEFT_HAND_ID, - hand_gesture_manager_lock.get_active_gestures(left_hand_skeleton.unwrap(), gestures_config.clone(), *LEFT_HAND_ID) - ); - trigger_hand_gesture_actions( - *RIGHT_HAND_ID, - hand_gesture_manager_lock.get_active_gestures(right_hand_skeleton.unwrap(), gestures_config.clone(), *RIGHT_HAND_ID) - ); + if tracking.hand_skeletons[0].is_some() { + trigger_hand_gesture_actions( + *LEFT_HAND_ID, + &hand_gesture_manager_lock.get_active_gestures( + tracking.hand_skeletons[0].unwrap(), + gestures_config.clone(), + *LEFT_HAND_ID, + ), + ); + } + if tracking.hand_skeletons[1].is_some() { + trigger_hand_gesture_actions( + *RIGHT_HAND_ID, + &hand_gesture_manager_lock.get_active_gestures( + tracking.hand_skeletons[1].unwrap(), + gestures_config.clone(), + *RIGHT_HAND_ID, + ), + ); + } drop(hand_gesture_manager_lock); } diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 30b3c874e5..e1f850be3a 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -11,7 +11,7 @@ use alvr_common::{ LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, - X_CLICK_ID, Y_CLICK_ID, + X_CLICK_ID, Y_CLICK_ID, warn, }; use alvr_session::HandGestureConfig; @@ -360,9 +360,6 @@ impl HandGestureManager { g.entering = false; } - // Default to maintaining state - let mut new_active_state = g.active; - // Get current time, for comparison let time_millis = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -378,7 +375,7 @@ impl HandGestureManager { if g.entering_since < time_millis - u128::from(in_delay) { g.last_activated = time_millis; g.entering = false; - new_active_state = true; + g.active = true; } } else { // Begin tracking entering state @@ -395,7 +392,7 @@ impl HandGestureManager { if g.exiting_since < time_millis - u128::from(out_delay) { g.last_deactivated = time_millis; g.exiting = false; - new_active_state = false; + g.active = false; } } else { // Begin tracking exiting state @@ -404,8 +401,6 @@ impl HandGestureManager { } } - g.active = new_active_state; - g.active } @@ -474,7 +469,7 @@ fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti } } -pub fn trigger_hand_gesture_actions(device_id: u64, gestures: Vec) { +pub fn trigger_hand_gesture_actions(device_id: u64, gestures: &Vec) { for gesture in gestures.iter() { // Active bind let active_bind = get_active_bind_for_gesture(device_id, gesture.id); @@ -499,9 +494,9 @@ pub fn trigger_hand_gesture_actions(device_id: u64, gestures: Vec) crate::SetButton( hover_bind.unwrap(), crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, + type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - scalar: gesture.hover.into(), + scalar: if gesture.active { gesture.hover } else { 0.0 }, }, }, ); From 55ef7e300027fc3ff2051d7fff67742421badafb Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 11:58:57 +0100 Subject: [PATCH 29/47] touch binds + correct click binds --- alvr/server/src/hand_gestures.rs | 135 ++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index e1f850be3a..c3c21f6348 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -11,7 +11,7 @@ use alvr_common::{ LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, - X_CLICK_ID, Y_CLICK_ID, warn, + X_CLICK_ID, Y_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, Y_TOUCH_ID, X_TOUCH_ID, RIGHT_TRIGGER_TOUCH_ID, B_TOUCH_ID, A_TOUCH_ID, }; use alvr_session::HandGestureConfig; @@ -36,6 +36,8 @@ fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { pub struct HandGesture { pub id: HandGestureId, pub active: bool, + pub clicked: bool, + pub touching: bool, pub hover: f32, } @@ -141,6 +143,20 @@ impl HandGestureManager { config.stop_delay, device_id, ), + clicked: self.test_gesture_dist( + thumb_tip, + thumb_rad, + index_tip, + index_rad, + pinch_min, + ), + touching: self.test_gesture_dist( + thumb_tip, + thumb_rad, + index_tip, + index_rad, + pinch_max, + ), hover: self.get_gesture_hover( thumb_tip, thumb_rad, index_tip, index_rad, pinch_min, pinch_max, ), @@ -161,6 +177,20 @@ impl HandGestureManager { config.stop_delay, device_id, ), + clicked: self.test_gesture_dist( + thumb_tip, + thumb_rad, + middle_tip, + middle_rad, + pinch_min, + ), + touching: self.test_gesture_dist( + thumb_tip, + thumb_rad, + middle_tip, + middle_rad, + pinch_max, + ), hover: self.get_gesture_hover( thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min, pinch_max, ), @@ -181,6 +211,20 @@ impl HandGestureManager { config.stop_delay, device_id, ), + clicked: self.test_gesture_dist( + thumb_tip, + thumb_rad, + ring_tip, + ring_rad, + pinch_min, + ), + touching: self.test_gesture_dist( + thumb_tip, + thumb_rad, + ring_tip, + ring_rad, + pinch_max, + ), hover: self.get_gesture_hover( thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min, pinch_max, ), @@ -201,6 +245,20 @@ impl HandGestureManager { config.stop_delay, device_id, ), + clicked: self.test_gesture_dist( + thumb_tip, + thumb_rad, + little_tip, + little_rad, + pinch_min, + ), + touching: self.test_gesture_dist( + thumb_tip, + thumb_rad, + little_tip, + little_rad, + pinch_max, + ), hover: self.get_gesture_hover( thumb_tip, thumb_rad, little_tip, little_rad, pinch_min, pinch_max, ), @@ -244,11 +302,13 @@ impl HandGestureManager { // Grip (NEEDS REFINING) let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; - let grip_active = grip_curl == 1.0; + let grip_active = grip_curl > 0.0; gestures.push(HandGesture { id: HandGestureId::GripCurl, active: grip_active, + clicked: grip_curl == 1.0, + touching: grip_curl > 0.0, hover: grip_curl, }); @@ -290,12 +350,16 @@ impl HandGestureManager { gestures.push(HandGesture { id: HandGestureId::ThumbCurl, - active: thumb_curl == 1.0, + active: thumb_curl >= 0.0, + touching: thumb_curl >= 0.0, + clicked: thumb_curl == 0.0, hover: thumb_curl, }); gestures.push(HandGesture { id: HandGestureId::JoystickX, active: joystick_contact, + touching: joystick_contact, + clicked: false, hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { joystick_pos.x } else { @@ -305,6 +369,8 @@ impl HandGestureManager { gestures.push(HandGesture { id: HandGestureId::JoystickY, active: joystick_contact, + touching: joystick_contact, + clicked: false, hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { joystick_pos.y } else { @@ -404,6 +470,18 @@ impl HandGestureManager { g.active } + fn test_gesture_dist( + &mut self, + first_anchor: Pose, + first_radius: f32, + second_anchor: Pose, + second_radius: f32, + activation_dist: f32, + ) -> bool { + first_anchor.position.distance(second_anchor.position) + < (activation_dist + first_radius + second_radius) + } + fn get_gesture_hover( &self, first_anchor: Pose, @@ -422,7 +500,7 @@ impl HandGestureManager { } } -fn get_active_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { +fn get_click_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { if device_id == *LEFT_HAND_ID { match gesture_id { HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_CLICK_ID), @@ -431,8 +509,6 @@ fn get_active_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opt HandGestureId::ThumbLittlePinch => Some(*MENU_CLICK_ID), HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_CLICK_ID), HandGestureId::ThumbCurl => Some(*LEFT_THUMBSTICK_CLICK_ID), - HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_TOUCH_ID), - HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_TOUCH_ID), _ => None, } } else { @@ -442,6 +518,26 @@ fn get_active_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opt HandGestureId::ThumbRingPinch => Some(*A_CLICK_ID), HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_CLICK_ID), HandGestureId::ThumbCurl => Some(*RIGHT_THUMBSTICK_CLICK_ID), + _ => None, + } + } +} + +fn get_touch_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { + if device_id == *LEFT_HAND_ID { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_TOUCH_ID), + HandGestureId::ThumbMiddlePinch => Some(*Y_TOUCH_ID), + HandGestureId::ThumbRingPinch => Some(*X_TOUCH_ID), + HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_TOUCH_ID), + HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_TOUCH_ID), + _ => None, + } + } else { + match gesture_id { + HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_TOUCH_ID), + HandGestureId::ThumbMiddlePinch => Some(*B_TOUCH_ID), + HandGestureId::ThumbRingPinch => Some(*A_TOUCH_ID), HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_TOUCH_ID), HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_TOUCH_ID), _ => None, @@ -471,16 +567,33 @@ fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti pub fn trigger_hand_gesture_actions(device_id: u64, gestures: &Vec) { for gesture in gestures.iter() { - // Active bind - let active_bind = get_active_bind_for_gesture(device_id, gesture.id); - if active_bind.is_some() { + + // Click bind + let click_bind = get_click_bind_for_gesture(device_id, gesture.id); + if click_bind.is_some() { + unsafe { + crate::SetButton( + click_bind.unwrap(), + crate::FfiButtonValue { + type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, + __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { + binary: (gesture.active && gesture.clicked).into(), + }, + }, + ); + } + } + + // Touch bind + let touch_bind = get_touch_bind_for_gesture(device_id, gesture.id); + if touch_bind.is_some() { unsafe { crate::SetButton( - active_bind.unwrap(), + touch_bind.unwrap(), crate::FfiButtonValue { type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - binary: gesture.active.into(), + binary: (gesture.active && gesture.touching).into(), }, }, ); From 6ff930ca061c4a23af1e234dab1596356081cf02 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 11:59:59 +0100 Subject: [PATCH 30/47] increase default stop delay --- alvr/session/src/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 7ef6592418..7310eaf9ce 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -1331,7 +1331,7 @@ pub fn session_settings_default() -> SettingsDefault { joystick_offset_vertical: 0.0, repeat_delay: 100, start_delay: 50, - stop_delay: 50 + stop_delay: 100 }, }, enable_skeleton: true, From 0b00d58adf0ebb9f3f90ea608b4f0384a5c33993 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:01:08 +0100 Subject: [PATCH 31/47] rename gesture delay settings --- alvr/server/src/hand_gestures.rs | 16 ++++++++-------- alvr/session/src/settings.rs | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index c3c21f6348..d6ec33f93b 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -139,8 +139,8 @@ impl HandGestureManager { index_rad, pinch_max, config.repeat_delay, - config.start_delay, - config.stop_delay, + config.activation_delay, + config.deactivation_delay, device_id, ), clicked: self.test_gesture_dist( @@ -173,8 +173,8 @@ impl HandGestureManager { middle_rad, pinch_max, config.repeat_delay, - config.start_delay, - config.stop_delay, + config.activation_delay, + config.deactivation_delay, device_id, ), clicked: self.test_gesture_dist( @@ -207,8 +207,8 @@ impl HandGestureManager { ring_rad, pinch_max, config.repeat_delay, - config.start_delay, - config.stop_delay, + config.activation_delay, + config.deactivation_delay, device_id, ), clicked: self.test_gesture_dist( @@ -241,8 +241,8 @@ impl HandGestureManager { little_rad, pinch_max, config.repeat_delay, - config.start_delay, - config.stop_delay, + config.activation_delay, + config.deactivation_delay, device_id, ), clicked: self.test_gesture_dist( diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 7310eaf9ce..71adc62554 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -765,14 +765,14 @@ pub struct HandGestureConfig { help = "How long the gesture must be continuously held before it is activated." ))] #[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")] - pub start_delay: u32, + pub activation_delay: u32, #[schema(flag = "real-time")] #[schema(strings( help = "How long the gesture must be continuously released before it is deactivated." ))] #[schema(gui(slider(min = 0, max = 1000)), suffix = "ms")] - pub stop_delay: u32, + pub deactivation_delay: u32, #[schema(flag = "real-time")] #[schema(strings( @@ -1330,8 +1330,8 @@ pub fn session_settings_default() -> SettingsDefault { joystick_offset_horizontal: 0.0, joystick_offset_vertical: 0.0, repeat_delay: 100, - start_delay: 50, - stop_delay: 100 + activation_delay: 50, + deactivation_delay: 100 }, }, enable_skeleton: true, From 0a11357176d5c0181ebb66962a75c38c4e47ee5a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:41:02 +0100 Subject: [PATCH 32/47] fix joystick click --- alvr/server/src/hand_gestures.rs | 84 ++++++++------------------------ 1 file changed, 20 insertions(+), 64 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index d6ec33f93b..690e018511 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -6,12 +6,13 @@ use std::{ use alvr_common::{ glam::{Quat, Vec2, Vec3}, - Pose, A_CLICK_ID, B_CLICK_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, - LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, - LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, - RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, - RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_VALUE_ID, - X_CLICK_ID, Y_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, Y_TOUCH_ID, X_TOUCH_ID, RIGHT_TRIGGER_TOUCH_ID, B_TOUCH_ID, A_TOUCH_ID, + Pose, A_CLICK_ID, A_TOUCH_ID, B_CLICK_ID, B_TOUCH_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, + LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, + LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, + LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, + RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, + RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_TOUCH_ID, RIGHT_TRIGGER_VALUE_ID, + X_CLICK_ID, X_TOUCH_ID, Y_CLICK_ID, Y_TOUCH_ID, }; use alvr_session::HandGestureConfig; @@ -143,20 +144,8 @@ impl HandGestureManager { config.deactivation_delay, device_id, ), - clicked: self.test_gesture_dist( - thumb_tip, - thumb_rad, - index_tip, - index_rad, - pinch_min, - ), - touching: self.test_gesture_dist( - thumb_tip, - thumb_rad, - index_tip, - index_rad, - pinch_max, - ), + clicked: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_min), + touching: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_max), hover: self.get_gesture_hover( thumb_tip, thumb_rad, index_tip, index_rad, pinch_min, pinch_max, ), @@ -177,20 +166,10 @@ impl HandGestureManager { config.deactivation_delay, device_id, ), - clicked: self.test_gesture_dist( - thumb_tip, - thumb_rad, - middle_tip, - middle_rad, - pinch_min, - ), - touching: self.test_gesture_dist( - thumb_tip, - thumb_rad, - middle_tip, - middle_rad, - pinch_max, - ), + clicked: self + .test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min), + touching: self + .test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_max), hover: self.get_gesture_hover( thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min, pinch_max, ), @@ -211,20 +190,8 @@ impl HandGestureManager { config.deactivation_delay, device_id, ), - clicked: self.test_gesture_dist( - thumb_tip, - thumb_rad, - ring_tip, - ring_rad, - pinch_min, - ), - touching: self.test_gesture_dist( - thumb_tip, - thumb_rad, - ring_tip, - ring_rad, - pinch_max, - ), + clicked: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min), + touching: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_max), hover: self.get_gesture_hover( thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min, pinch_max, ), @@ -245,20 +212,10 @@ impl HandGestureManager { config.deactivation_delay, device_id, ), - clicked: self.test_gesture_dist( - thumb_tip, - thumb_rad, - little_tip, - little_rad, - pinch_min, - ), - touching: self.test_gesture_dist( - thumb_tip, - thumb_rad, - little_tip, - little_rad, - pinch_max, - ), + clicked: self + .test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_min), + touching: self + .test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_max), hover: self.get_gesture_hover( thumb_tip, thumb_rad, little_tip, little_rad, pinch_min, pinch_max, ), @@ -352,7 +309,7 @@ impl HandGestureManager { id: HandGestureId::ThumbCurl, active: thumb_curl >= 0.0, touching: thumb_curl >= 0.0, - clicked: thumb_curl == 0.0, + clicked: thumb_curl >= 0.5, hover: thumb_curl, }); gestures.push(HandGesture { @@ -567,7 +524,6 @@ fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti pub fn trigger_hand_gesture_actions(device_id: u64, gestures: &Vec) { for gesture in gestures.iter() { - // Click bind let click_bind = get_click_bind_for_gesture(device_id, gesture.id); if click_bind.is_some() { From 8acbd6a1d97e245a1b9d5c562fc41790e859ea4f Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:42:30 +0100 Subject: [PATCH 33/47] fmt --- alvr/server/src/tracking.rs | 1 - alvr/session/src/settings.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/alvr/server/src/tracking.rs b/alvr/server/src/tracking.rs index 61606e9091..6488736f10 100644 --- a/alvr/server/src/tracking.rs +++ b/alvr/server/src/tracking.rs @@ -65,7 +65,6 @@ impl TrackingManager { TrackingManager { last_head_pose: Pose::default(), recentering_origin: Pose::default(), - } } diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 71adc62554..c77ec09281 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -1331,7 +1331,7 @@ pub fn session_settings_default() -> SettingsDefault { joystick_offset_vertical: 0.0, repeat_delay: 100, activation_delay: 50, - deactivation_delay: 100 + deactivation_delay: 100, }, }, enable_skeleton: true, From c8f4a8ad6da98eee29180edfde667e9b2a204818 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:40:41 +0100 Subject: [PATCH 34/47] I DID IT !!!! JOYSTICK !!!!!!! --- alvr/server/src/hand_gestures.rs | 101 +++++++++++++++++-------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 690e018511..db9749ba92 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -5,7 +5,7 @@ use std::{ }; use alvr_common::{ - glam::{Quat, Vec2, Vec3}, + glam::{Vec2, Vec3}, Pose, A_CLICK_ID, A_TOUCH_ID, B_CLICK_ID, B_TOUCH_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, @@ -19,17 +19,8 @@ use alvr_session::HandGestureConfig; fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { Pose { - orientation: Quat::from_xyzw( - a.orientation.x * (1.0 - fac) + (b.position.x * fac), - a.orientation.y * (1.0 - fac) + (b.orientation.y * fac), - a.orientation.z * (1.0 - fac) + (b.orientation.z * fac), - a.orientation.w * (1.0 - fac) + (b.orientation.w * fac), - ), - position: Vec3 { - x: a.position.x * (1.0 - fac) + (b.position.x * fac), - y: a.position.y * (1.0 - fac) + (b.position.y * fac), - z: a.position.z * (1.0 - fac) + (b.position.z * fac), - }, + orientation: a.orientation.lerp(b.orientation, fac), + position: a.position.lerp(b.position, fac), } } @@ -110,15 +101,14 @@ impl HandGestureManager { let curl_max = config.curl_trigger_distance * 0.01; let palm: Pose = gj[0]; - let thumb_proximal: Pose = gj[3]; let thumb_tip: Pose = gj[5]; let index_metacarpal: Pose = gj[6]; let index_proximal: Pose = gj[7]; let index_intermediate: Pose = gj[8]; + let index_distal: Pose = gj[9]; let index_tip: Pose = gj[10]; let middle_metacarpal: Pose = gj[11]; let middle_proximal: Pose = gj[12]; - let middle_intermediate: Pose = gj[13]; let middle_tip: Pose = gj[15]; let ring_metacarpal: Pose = gj[16]; let ring_proximal: Pose = gj[17]; @@ -257,7 +247,7 @@ impl HandGestureManager { curl_max, ); - // Grip (NEEDS REFINING) + // Grip let grip_curl = (middle_curl + ring_curl + little_curl) / 3.0; let grip_active = grip_curl > 0.0; @@ -271,36 +261,39 @@ impl HandGestureManager { // Joystick (NEEDS REFINING) let joystick_range = 0.01; - let joystick_center = index_intermediate.position.lerp(index_tip.position, 0.25); + let joystick_center = lerp_pose(index_intermediate, index_distal, 0.5); - let joystick_up = (joystick_center - - middle_intermediate.position.lerp(middle_tip.position, 0.25)) - .normalize() - * joystick_range; - - let joystick_vertical_vec = - (joystick_center - thumb_proximal.position).normalize() * joystick_range; + let joystick_up = joystick_center + .orientation + .mul_vec3(if device_id == *LEFT_HAND_ID { + Vec3::X + } else { + Vec3::NEG_X + }); let joystick_horizontal_vec = - joystick_vertical_vec.cross(joystick_up).normalize() * joystick_range; - - let joystick_vertical = (thumb_tip.position - joystick_center - + joystick_vertical_vec / 2.0) - .dot(joystick_vertical_vec) - / joystick_vertical_vec.length() - + config.joystick_offset_vertical * 0.01; - let joystick_horizontal = (thumb_tip.position - joystick_center) - .dot(joystick_horizontal_vec) - / joystick_horizontal_vec.length() - + config.joystick_offset_horizontal * 0.01; - - let joystick_pos = Vec2 { - x: (joystick_horizontal / joystick_range).clamp(-1.0, 1.0), - y: (joystick_vertical / joystick_range).clamp(-1.0, 1.0), - }; + index_intermediate + .orientation + .mul_vec3(if device_id == *LEFT_HAND_ID { + Vec3::Y + } else { + Vec3::NEG_Y + }); + let joystick_vertical_vec = index_intermediate.orientation.mul_vec3(Vec3::Z); + + let joystick_pos = self.get_joystick_values( + joystick_center, + thumb_tip, + joystick_range, + joystick_horizontal_vec, + joystick_vertical_vec, + config.joystick_offset_horizontal * 0.01, + config.joystick_offset_vertical * 0.01, + ); let joystick_contact = index_curl >= 0.75 && grip_curl > 0.5 - && joystick_center.distance(thumb_tip.position) <= joystick_range * 5.0 - && (thumb_tip.position - joystick_center).dot(joystick_up) / joystick_up.length() + && joystick_center.position.distance(thumb_tip.position) <= joystick_range * 5.0 + && (thumb_tip.position - joystick_center.position).dot(joystick_up) + / joystick_up.length() <= joystick_range * 3.0; let joystick_deadzone: f32 = config.joystick_deadzone * 0.01; @@ -317,7 +310,7 @@ impl HandGestureManager { active: joystick_contact, touching: joystick_contact, clicked: false, - hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { + hover: if joystick_contact && joystick_pos.x.abs() >= joystick_deadzone { joystick_pos.x } else { 0.0 @@ -328,7 +321,7 @@ impl HandGestureManager { active: joystick_contact, touching: joystick_contact, clicked: false, - hover: if joystick_contact && joystick_pos.y >= joystick_deadzone { + hover: if joystick_contact && joystick_pos.y.abs() >= joystick_deadzone { joystick_pos.y } else { 0.0 @@ -428,7 +421,7 @@ impl HandGestureManager { } fn test_gesture_dist( - &mut self, + &self, first_anchor: Pose, first_radius: f32, second_anchor: Pose, @@ -455,6 +448,26 @@ impl HandGestureManager { / (max_dist + first_radius + second_radius)) .clamp(0.0, 1.0) } + + fn get_joystick_values( + &self, + center: Pose, + anchor: Pose, + joy_radius: f32, + hori_vec: Vec3, + vert_vec: Vec3, + offset_hori: f32, + offset_vert: f32, + ) -> Vec2 { + let x = (anchor.position - center.position).dot(hori_vec) / hori_vec.length() + offset_hori; + + let y = (anchor.position - center.position).dot(vert_vec) / vert_vec.length() + offset_vert; + + Vec2 { + x: (x / joy_radius).clamp(-1.0, 1.0), + y: (y / joy_radius).clamp(-1.0, 1.0), + } + } } fn get_click_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Option { From 21041a70672f20e2cabe3d8f69120b737f6dbdb0 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:41:40 +0100 Subject: [PATCH 35/47] rename hover to value --- alvr/server/src/hand_gestures.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index db9749ba92..af032e2048 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -30,7 +30,7 @@ pub struct HandGesture { pub active: bool, pub clicked: bool, pub touching: bool, - pub hover: f32, + pub value: f32, } pub struct GestureAction { @@ -136,7 +136,7 @@ impl HandGestureManager { ), clicked: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_min), touching: self.test_gesture_dist(thumb_tip, thumb_rad, index_tip, index_rad, pinch_max), - hover: self.get_gesture_hover( + value: self.get_gesture_hover( thumb_tip, thumb_rad, index_tip, index_rad, pinch_min, pinch_max, ), }); @@ -160,7 +160,7 @@ impl HandGestureManager { .test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min), touching: self .test_gesture_dist(thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_max), - hover: self.get_gesture_hover( + value: self.get_gesture_hover( thumb_tip, thumb_rad, middle_tip, middle_rad, pinch_min, pinch_max, ), }); @@ -182,7 +182,7 @@ impl HandGestureManager { ), clicked: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min), touching: self.test_gesture_dist(thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_max), - hover: self.get_gesture_hover( + value: self.get_gesture_hover( thumb_tip, thumb_rad, ring_tip, ring_rad, pinch_min, pinch_max, ), }); @@ -206,7 +206,7 @@ impl HandGestureManager { .test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_min), touching: self .test_gesture_dist(thumb_tip, thumb_rad, little_tip, little_rad, pinch_max), - hover: self.get_gesture_hover( + value: self.get_gesture_hover( thumb_tip, thumb_rad, little_tip, little_rad, pinch_min, pinch_max, ), }); @@ -256,10 +256,10 @@ impl HandGestureManager { active: grip_active, clicked: grip_curl == 1.0, touching: grip_curl > 0.0, - hover: grip_curl, + value: grip_curl, }); - // Joystick (NEEDS REFINING) + // Joystick let joystick_range = 0.01; let joystick_center = lerp_pose(index_intermediate, index_distal, 0.5); @@ -303,14 +303,14 @@ impl HandGestureManager { active: thumb_curl >= 0.0, touching: thumb_curl >= 0.0, clicked: thumb_curl >= 0.5, - hover: thumb_curl, + value: thumb_curl, }); gestures.push(HandGesture { id: HandGestureId::JoystickX, active: joystick_contact, touching: joystick_contact, clicked: false, - hover: if joystick_contact && joystick_pos.x.abs() >= joystick_deadzone { + value: if joystick_contact && joystick_pos.x.abs() >= joystick_deadzone { joystick_pos.x } else { 0.0 @@ -321,7 +321,7 @@ impl HandGestureManager { active: joystick_contact, touching: joystick_contact, clicked: false, - hover: if joystick_contact && joystick_pos.y.abs() >= joystick_deadzone { + value: if joystick_contact && joystick_pos.y.abs() >= joystick_deadzone { joystick_pos.y } else { 0.0 @@ -578,7 +578,7 @@ pub fn trigger_hand_gesture_actions(device_id: u64, gestures: &Vec) crate::FfiButtonValue { type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - scalar: if gesture.active { gesture.hover } else { 0.0 }, + scalar: if gesture.active { gesture.value } else { 0.0 }, }, }, ); From 5563f97d106544b1c159fe63e9ccd75709daf990 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:52:35 +0100 Subject: [PATCH 36/47] remove hand tracking toggle --- alvr/server/src/connection.rs | 54 ++++++++++++++++------------------- alvr/session/src/settings.rs | 40 ++++++++++++-------------- 2 files changed, 42 insertions(+), 52 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index c25f2f8c83..4a0eee5aab 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -753,45 +753,39 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { // Handle hand gestures if let Switch::Enabled(controllers_config) = &settings.headset.controllers { - if let Switch::Enabled(hand_tracking_config) = - &controllers_config.hand_tracking + if let Switch::Enabled(gestures_config) = + &controllers_config.hand_tracking.use_gestures { - if let Switch::Enabled(gestures_config) = - &hand_tracking_config.use_gestures - { - let mut hand_gesture_manager_lock = hand_gesture_manager.lock(); - - if tracking.hand_skeletons[0].is_some() { - trigger_hand_gesture_actions( + let mut hand_gesture_manager_lock = hand_gesture_manager.lock(); + + if tracking.hand_skeletons[0].is_some() { + trigger_hand_gesture_actions( + *LEFT_HAND_ID, + &hand_gesture_manager_lock.get_active_gestures( + tracking.hand_skeletons[0].unwrap(), + gestures_config.clone(), *LEFT_HAND_ID, - &hand_gesture_manager_lock.get_active_gestures( - tracking.hand_skeletons[0].unwrap(), - gestures_config.clone(), - *LEFT_HAND_ID, - ), - ); - } - if tracking.hand_skeletons[1].is_some() { - trigger_hand_gesture_actions( + ), + ); + } + if tracking.hand_skeletons[1].is_some() { + trigger_hand_gesture_actions( + *RIGHT_HAND_ID, + &hand_gesture_manager_lock.get_active_gestures( + tracking.hand_skeletons[1].unwrap(), + gestures_config.clone(), *RIGHT_HAND_ID, - &hand_gesture_manager_lock.get_active_gestures( - tracking.hand_skeletons[1].unwrap(), - gestures_config.clone(), - *RIGHT_HAND_ID, - ), - ); - } - - drop(hand_gesture_manager_lock); + ), + ); } + + drop(hand_gesture_manager_lock); } } let mut hand_skeletons_enabled = false; if let Switch::Enabled(controllers) = &config.controllers { - if let Switch::Enabled(hand_tracking) = &controllers.hand_tracking { - hand_skeletons_enabled = hand_tracking.enable_skeleton; - } + hand_skeletons_enabled = controllers.hand_tracking.enable_skeleton; } unsafe { diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index c77ec09281..1cd2d4c147 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -659,8 +659,7 @@ pub struct ControllersConfig { pub button_mapping_config: AutomaticButtonMappingConfig, - #[schema(strings(help = "Turning this off will disable hand-tracking."))] - pub hand_tracking: Switch, + pub hand_tracking: HandTrackingConfig, #[schema(strings( display_name = "Prediction", @@ -1314,28 +1313,25 @@ pub fn session_settings_default() -> SettingsDefault { }, force_threshold: 0.8, }, - hand_tracking: SwitchDefault { - enabled: true, - content: HandTrackingConfigDefault { - gui_collapsed: true, - use_gestures: SwitchDefault { - enabled: true, - content: HandGestureConfigDefault { - gui_collapsed: true, - pinch_touch_distance: 0.0, - pinch_trigger_distance: 0.25, - curl_touch_distance: 2.0, - curl_trigger_distance: 2.5, - joystick_deadzone: 25.0, - joystick_offset_horizontal: 0.0, - joystick_offset_vertical: 0.0, - repeat_delay: 100, - activation_delay: 50, - deactivation_delay: 100, - }, + hand_tracking: HandTrackingConfigDefault { + gui_collapsed: true, + use_gestures: SwitchDefault { + enabled: true, + content: HandGestureConfigDefault { + gui_collapsed: true, + pinch_touch_distance: 0.0, + pinch_trigger_distance: 0.25, + curl_touch_distance: 2.0, + curl_trigger_distance: 2.5, + joystick_deadzone: 25.0, + joystick_offset_horizontal: 0.0, + joystick_offset_vertical: 0.0, + repeat_delay: 100, + activation_delay: 50, + deactivation_delay: 100, }, - enable_skeleton: true, }, + enable_skeleton: true, }, steamvr_pipeline_frames: 3.0, linear_velocity_cutoff: 0.05, From 9e7163c89127c9bd7f50ad94355a2c036ffddd11 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:34:13 +0100 Subject: [PATCH 37/47] handed abxy IDs --- alvr/server/src/hand_gestures.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index af032e2048..6ed459800f 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -6,13 +6,13 @@ use std::{ use alvr_common::{ glam::{Vec2, Vec3}, - Pose, A_CLICK_ID, A_TOUCH_ID, B_CLICK_ID, B_TOUCH_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, + Pose, RIGHT_A_CLICK_ID, RIGHT_A_TOUCH_ID, RIGHT_B_CLICK_ID, RIGHT_B_TOUCH_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, - LEFT_TRIGGER_VALUE_ID, MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, + LEFT_TRIGGER_VALUE_ID, LEFT_MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_TOUCH_ID, RIGHT_TRIGGER_VALUE_ID, - X_CLICK_ID, X_TOUCH_ID, Y_CLICK_ID, Y_TOUCH_ID, + LEFT_X_CLICK_ID, LEFT_X_TOUCH_ID, LEFT_Y_CLICK_ID, LEFT_Y_TOUCH_ID, }; use alvr_session::HandGestureConfig; @@ -474,9 +474,9 @@ fn get_click_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti if device_id == *LEFT_HAND_ID { match gesture_id { HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_CLICK_ID), - HandGestureId::ThumbMiddlePinch => Some(*Y_CLICK_ID), - HandGestureId::ThumbRingPinch => Some(*X_CLICK_ID), - HandGestureId::ThumbLittlePinch => Some(*MENU_CLICK_ID), + HandGestureId::ThumbMiddlePinch => Some(*LEFT_Y_CLICK_ID), + HandGestureId::ThumbRingPinch => Some(*LEFT_X_CLICK_ID), + HandGestureId::ThumbLittlePinch => Some(*LEFT_MENU_CLICK_ID), HandGestureId::GripCurl => Some(*LEFT_SQUEEZE_CLICK_ID), HandGestureId::ThumbCurl => Some(*LEFT_THUMBSTICK_CLICK_ID), _ => None, @@ -484,8 +484,8 @@ fn get_click_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti } else { match gesture_id { HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_CLICK_ID), - HandGestureId::ThumbMiddlePinch => Some(*B_CLICK_ID), - HandGestureId::ThumbRingPinch => Some(*A_CLICK_ID), + HandGestureId::ThumbMiddlePinch => Some(*RIGHT_B_CLICK_ID), + HandGestureId::ThumbRingPinch => Some(*RIGHT_A_CLICK_ID), HandGestureId::GripCurl => Some(*RIGHT_SQUEEZE_CLICK_ID), HandGestureId::ThumbCurl => Some(*RIGHT_THUMBSTICK_CLICK_ID), _ => None, @@ -497,8 +497,8 @@ fn get_touch_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti if device_id == *LEFT_HAND_ID { match gesture_id { HandGestureId::ThumbIndexPinch => Some(*LEFT_TRIGGER_TOUCH_ID), - HandGestureId::ThumbMiddlePinch => Some(*Y_TOUCH_ID), - HandGestureId::ThumbRingPinch => Some(*X_TOUCH_ID), + HandGestureId::ThumbMiddlePinch => Some(*LEFT_Y_TOUCH_ID), + HandGestureId::ThumbRingPinch => Some(*LEFT_X_TOUCH_ID), HandGestureId::JoystickX => Some(*LEFT_THUMBSTICK_TOUCH_ID), HandGestureId::JoystickY => Some(*LEFT_THUMBSTICK_TOUCH_ID), _ => None, @@ -506,8 +506,8 @@ fn get_touch_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti } else { match gesture_id { HandGestureId::ThumbIndexPinch => Some(*RIGHT_TRIGGER_TOUCH_ID), - HandGestureId::ThumbMiddlePinch => Some(*B_TOUCH_ID), - HandGestureId::ThumbRingPinch => Some(*A_TOUCH_ID), + HandGestureId::ThumbMiddlePinch => Some(*RIGHT_B_TOUCH_ID), + HandGestureId::ThumbRingPinch => Some(*RIGHT_A_TOUCH_ID), HandGestureId::JoystickX => Some(*RIGHT_THUMBSTICK_TOUCH_ID), HandGestureId::JoystickY => Some(*RIGHT_THUMBSTICK_TOUCH_ID), _ => None, From 3ff070d546d7cc2a6d03989d652f9feeea7f9142 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:38:15 +0100 Subject: [PATCH 38/47] format --- alvr/server/src/hand_gestures.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 6ed459800f..eaff0c2ce3 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -6,13 +6,13 @@ use std::{ use alvr_common::{ glam::{Vec2, Vec3}, - Pose, RIGHT_A_CLICK_ID, RIGHT_A_TOUCH_ID, RIGHT_B_CLICK_ID, RIGHT_B_TOUCH_ID, LEFT_HAND_ID, LEFT_SQUEEZE_CLICK_ID, - LEFT_SQUEEZE_VALUE_ID, LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, - LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, - LEFT_TRIGGER_VALUE_ID, LEFT_MENU_CLICK_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, + Pose, LEFT_HAND_ID, LEFT_MENU_CLICK_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, + LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, + LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, LEFT_TRIGGER_VALUE_ID, LEFT_X_CLICK_ID, + LEFT_X_TOUCH_ID, LEFT_Y_CLICK_ID, LEFT_Y_TOUCH_ID, RIGHT_A_CLICK_ID, RIGHT_A_TOUCH_ID, + RIGHT_B_CLICK_ID, RIGHT_B_TOUCH_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_TOUCH_ID, RIGHT_TRIGGER_VALUE_ID, - LEFT_X_CLICK_ID, LEFT_X_TOUCH_ID, LEFT_Y_CLICK_ID, LEFT_Y_TOUCH_ID, }; use alvr_session::HandGestureConfig; From de159284dee3dc25699eb0baed9cdde839a55f4a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:05:09 +0100 Subject: [PATCH 39/47] use button mapping manager (slightly broken?) --- alvr/server/src/connection.rs | 21 ++++++++++++++++ alvr/server/src/hand_gestures.rs | 41 ++++++-------------------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 4a0eee5aab..fb6cc19370 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -649,6 +649,25 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { let tracking_receive_thread = thread::spawn({ let tracking_manager = Arc::clone(&tracking_manager); let hand_gesture_manager = Arc::clone(&hand_gesture_manager); + + let mut gestures_button_mapping_manager = if let Switch::Enabled(config) = + &SERVER_DATA_MANAGER.read().settings().headset.controllers + { + if let Switch::Enabled(_) = &config.hand_tracking.use_gestures { + Some(ButtonMappingManager::new_automatic( + &CONTROLLER_PROFILE_INFO + .get(&alvr_common::hash_string(QUEST_CONTROLLER_PROFILE_PATH)) + .unwrap() + .button_set, + &config.button_mapping_config, + )) + } else { + None + } + } else { + None + }; + move || { let mut face_tracking_sink = settings @@ -760,6 +779,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { if tracking.hand_skeletons[0].is_some() { trigger_hand_gesture_actions( + gestures_button_mapping_manager.as_mut().unwrap(), *LEFT_HAND_ID, &hand_gesture_manager_lock.get_active_gestures( tracking.hand_skeletons[0].unwrap(), @@ -770,6 +790,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { } if tracking.hand_skeletons[1].is_some() { trigger_hand_gesture_actions( + gestures_button_mapping_manager.as_mut().unwrap(), *RIGHT_HAND_ID, &hand_gesture_manager_lock.get_active_gestures( tracking.hand_skeletons[1].unwrap(), diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index eaff0c2ce3..d85b95da76 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -15,8 +15,11 @@ use alvr_common::{ RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_TOUCH_ID, RIGHT_TRIGGER_VALUE_ID, }; +use alvr_packets::ButtonValue; use alvr_session::HandGestureConfig; +use crate::input_mapping::ButtonMappingManager; + fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { Pose { orientation: a.orientation.lerp(b.orientation, fac), @@ -535,54 +538,24 @@ fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti } } -pub fn trigger_hand_gesture_actions(device_id: u64, gestures: &Vec) { +pub fn trigger_hand_gesture_actions(button_mapping_manager: &mut ButtonMappingManager, device_id: u64, gestures: &Vec) { for gesture in gestures.iter() { // Click bind let click_bind = get_click_bind_for_gesture(device_id, gesture.id); if click_bind.is_some() { - unsafe { - crate::SetButton( - click_bind.unwrap(), - crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, - __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - binary: (gesture.active && gesture.clicked).into(), - }, - }, - ); - } + button_mapping_manager.report_button(click_bind.unwrap(), ButtonValue::Binary(gesture.active && gesture.clicked)); } // Touch bind let touch_bind = get_touch_bind_for_gesture(device_id, gesture.id); if touch_bind.is_some() { - unsafe { - crate::SetButton( - touch_bind.unwrap(), - crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_BINARY, - __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - binary: (gesture.active && gesture.touching).into(), - }, - }, - ); - } + button_mapping_manager.report_button(touch_bind.unwrap(), ButtonValue::Binary(gesture.active && gesture.touching)); } // Hover bind let hover_bind = get_hover_bind_for_gesture(device_id, gesture.id); if hover_bind.is_some() { - unsafe { - crate::SetButton( - hover_bind.unwrap(), - crate::FfiButtonValue { - type_: crate::FfiButtonType_BUTTON_TYPE_SCALAR, - __bindgen_anon_1: crate::FfiButtonValue__bindgen_ty_1 { - scalar: if gesture.active { gesture.value } else { 0.0 }, - }, - }, - ); - } + button_mapping_manager.report_button(hover_bind.unwrap(), ButtonValue::Scalar(if gesture.active { gesture.value } else { 0.0 })); } } } From 9ce879bbcd76254f6d8f1868d49253819852aecc Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:08:09 +0100 Subject: [PATCH 40/47] format --- alvr/server/src/hand_gestures.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index d85b95da76..17017f21c4 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -538,24 +538,37 @@ fn get_hover_bind_for_gesture(device_id: u64, gesture_id: HandGestureId) -> Opti } } -pub fn trigger_hand_gesture_actions(button_mapping_manager: &mut ButtonMappingManager, device_id: u64, gestures: &Vec) { +pub fn trigger_hand_gesture_actions( + button_mapping_manager: &mut ButtonMappingManager, + device_id: u64, + gestures: &Vec, +) { for gesture in gestures.iter() { // Click bind let click_bind = get_click_bind_for_gesture(device_id, gesture.id); if click_bind.is_some() { - button_mapping_manager.report_button(click_bind.unwrap(), ButtonValue::Binary(gesture.active && gesture.clicked)); + button_mapping_manager.report_button( + click_bind.unwrap(), + ButtonValue::Binary(gesture.active && gesture.clicked), + ); } // Touch bind let touch_bind = get_touch_bind_for_gesture(device_id, gesture.id); if touch_bind.is_some() { - button_mapping_manager.report_button(touch_bind.unwrap(), ButtonValue::Binary(gesture.active && gesture.touching)); + button_mapping_manager.report_button( + touch_bind.unwrap(), + ButtonValue::Binary(gesture.active && gesture.touching), + ); } // Hover bind let hover_bind = get_hover_bind_for_gesture(device_id, gesture.id); if hover_bind.is_some() { - button_mapping_manager.report_button(hover_bind.unwrap(), ButtonValue::Scalar(if gesture.active { gesture.value } else { 0.0 })); + button_mapping_manager.report_button( + hover_bind.unwrap(), + ButtonValue::Scalar(if gesture.active { gesture.value } else { 0.0 }), + ); } } } From 808162c8cfb76ad42ee74da2b0154aa5c424d06a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sun, 10 Sep 2023 11:10:44 +0100 Subject: [PATCH 41/47] review response :) --- alvr/server/src/connection.rs | 45 ++++++++++++++++++++------------ alvr/server/src/hand_gestures.rs | 41 +++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index fb6cc19370..4943a8c93f 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -1,7 +1,7 @@ use crate::{ bitrate::BitrateManager, face_tracking::FaceTrackingSink, - hand_gestures::{trigger_hand_gesture_actions, HandGestureManager}, + hand_gestures::{trigger_hand_gesture_actions, HandGestureManager, HAND_GESTURE_PROFILE_INFO}, haptics, input_mapping::ButtonMappingManager, sockets::WelcomeSocket, @@ -38,7 +38,7 @@ use std::{ io::Write, net::IpAddr, process::Command, - ptr::{self}, + ptr, sync::{ atomic::{AtomicBool, Ordering}, mpsc::{self, RecvTimeoutError, SyncSender, TrySendError}, @@ -643,8 +643,18 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { thread::spawn(|| ()) }; - let tracking_manager = Arc::new(Mutex::new(TrackingManager::new())); - let hand_gesture_manager = Arc::new(Mutex::new(HandGestureManager::new())); + let tracking_manager: Arc< + alvr_common::parking_lot::lock_api::Mutex< + alvr_common::parking_lot::RawMutex, + TrackingManager, + >, + > = Arc::new(Mutex::new(TrackingManager::new())); + let hand_gesture_manager: Arc< + alvr_common::parking_lot::lock_api::Mutex< + alvr_common::parking_lot::RawMutex, + HandGestureManager, + >, + > = Arc::new(Mutex::new(HandGestureManager::new())); let tracking_receive_thread = thread::spawn({ let tracking_manager = Arc::clone(&tracking_manager); @@ -655,10 +665,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { { if let Switch::Enabled(_) = &config.hand_tracking.use_gestures { Some(ButtonMappingManager::new_automatic( - &CONTROLLER_PROFILE_INFO - .get(&alvr_common::hash_string(QUEST_CONTROLLER_PROFILE_PATH)) - .unwrap() - .button_set, + &HAND_GESTURE_PROFILE_INFO.button_set, &config.button_mapping_config, )) } else { @@ -693,15 +700,15 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { return; }; - let data_manager_lock = SERVER_DATA_MANAGER.read(); - let config = &data_manager_lock.settings().headset; - let mut tracking_manager_lock = tracking_manager.lock(); let motions; let left_hand_skeleton; let right_hand_skeleton; { + let data_manager_lock = SERVER_DATA_MANAGER.read(); + let config = &data_manager_lock.settings().headset; + motions = tracking_manager_lock.transform_motions( config, &tracking.device_motions, @@ -726,6 +733,7 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .unwrap_or_default(); { + let data_manager_lock = SERVER_DATA_MANAGER.read(); if data_manager_lock.settings().logging.log_tracking { alvr_events::send_event(EventType::Tracking(Box::new(TrackingEvent { head_motion: motions @@ -799,15 +807,18 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { ), ); } - - drop(hand_gesture_manager_lock); } } - let mut hand_skeletons_enabled = false; - if let Switch::Enabled(controllers) = &config.controllers { - hand_skeletons_enabled = controllers.hand_tracking.enable_skeleton; - } + let data_manager_lock = SERVER_DATA_MANAGER.read(); + let config = &data_manager_lock.settings().headset; + + let hand_skeletons_enabled = + if let Switch::Enabled(controllers) = &config.controllers { + controllers.hand_tracking.enable_skeleton + } else { + false + }; unsafe { crate::SetTracking( diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 17017f21c4..ba413bee94 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -5,14 +5,7 @@ use std::{ }; use alvr_common::{ - glam::{Vec2, Vec3}, - Pose, LEFT_HAND_ID, LEFT_MENU_CLICK_ID, LEFT_SQUEEZE_CLICK_ID, LEFT_SQUEEZE_VALUE_ID, - LEFT_THUMBSTICK_CLICK_ID, LEFT_THUMBSTICK_TOUCH_ID, LEFT_THUMBSTICK_X_ID, LEFT_THUMBSTICK_Y_ID, - LEFT_TRIGGER_CLICK_ID, LEFT_TRIGGER_TOUCH_ID, LEFT_TRIGGER_VALUE_ID, LEFT_X_CLICK_ID, - LEFT_X_TOUCH_ID, LEFT_Y_CLICK_ID, LEFT_Y_TOUCH_ID, RIGHT_A_CLICK_ID, RIGHT_A_TOUCH_ID, - RIGHT_B_CLICK_ID, RIGHT_B_TOUCH_ID, RIGHT_SQUEEZE_CLICK_ID, RIGHT_SQUEEZE_VALUE_ID, - RIGHT_THUMBSTICK_CLICK_ID, RIGHT_THUMBSTICK_TOUCH_ID, RIGHT_THUMBSTICK_X_ID, - RIGHT_THUMBSTICK_Y_ID, RIGHT_TRIGGER_CLICK_ID, RIGHT_TRIGGER_TOUCH_ID, RIGHT_TRIGGER_VALUE_ID, + glam::{Vec2, Vec3}, *, once_cell::sync::Lazy, }; use alvr_packets::ButtonValue; @@ -27,6 +20,38 @@ fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { } } +pub static HAND_GESTURE_PROFILE_INFO: Lazy = Lazy::new(|| InteractionProfileInfo { + path: QUEST_CONTROLLER_PROFILE_PATH, + button_set: [ + *LEFT_X_CLICK_ID, + *LEFT_X_TOUCH_ID, + *LEFT_Y_CLICK_ID, + *LEFT_Y_TOUCH_ID, + *LEFT_MENU_CLICK_ID, + *LEFT_SQUEEZE_VALUE_ID, + *LEFT_TRIGGER_VALUE_ID, + *LEFT_TRIGGER_TOUCH_ID, + *LEFT_THUMBSTICK_X_ID, + *LEFT_THUMBSTICK_Y_ID, + *LEFT_THUMBSTICK_CLICK_ID, + *LEFT_THUMBSTICK_TOUCH_ID, + *RIGHT_A_CLICK_ID, + *RIGHT_A_TOUCH_ID, + *RIGHT_B_CLICK_ID, + *RIGHT_B_TOUCH_ID, + *RIGHT_SQUEEZE_VALUE_ID, + *RIGHT_TRIGGER_VALUE_ID, + *RIGHT_TRIGGER_TOUCH_ID, + *RIGHT_THUMBSTICK_X_ID, + *RIGHT_THUMBSTICK_Y_ID, + *RIGHT_THUMBSTICK_CLICK_ID, + *RIGHT_THUMBSTICK_TOUCH_ID, + ] + .into_iter() + .collect(), +} +); + #[derive(Debug, Clone)] pub struct HandGesture { pub id: HandGestureId, From 6f307b1ae4659ff3be0d7c17f7f99d3cd626858f Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sun, 10 Sep 2023 11:29:45 +0100 Subject: [PATCH 42/47] fmt --- alvr/server/src/hand_gestures.rs | 66 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index ba413bee94..51c6c63771 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -5,7 +5,9 @@ use std::{ }; use alvr_common::{ - glam::{Vec2, Vec3}, *, once_cell::sync::Lazy, + glam::{Vec2, Vec3}, + once_cell::sync::Lazy, + *, }; use alvr_packets::ButtonValue; @@ -20,37 +22,37 @@ fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { } } -pub static HAND_GESTURE_PROFILE_INFO: Lazy = Lazy::new(|| InteractionProfileInfo { - path: QUEST_CONTROLLER_PROFILE_PATH, - button_set: [ - *LEFT_X_CLICK_ID, - *LEFT_X_TOUCH_ID, - *LEFT_Y_CLICK_ID, - *LEFT_Y_TOUCH_ID, - *LEFT_MENU_CLICK_ID, - *LEFT_SQUEEZE_VALUE_ID, - *LEFT_TRIGGER_VALUE_ID, - *LEFT_TRIGGER_TOUCH_ID, - *LEFT_THUMBSTICK_X_ID, - *LEFT_THUMBSTICK_Y_ID, - *LEFT_THUMBSTICK_CLICK_ID, - *LEFT_THUMBSTICK_TOUCH_ID, - *RIGHT_A_CLICK_ID, - *RIGHT_A_TOUCH_ID, - *RIGHT_B_CLICK_ID, - *RIGHT_B_TOUCH_ID, - *RIGHT_SQUEEZE_VALUE_ID, - *RIGHT_TRIGGER_VALUE_ID, - *RIGHT_TRIGGER_TOUCH_ID, - *RIGHT_THUMBSTICK_X_ID, - *RIGHT_THUMBSTICK_Y_ID, - *RIGHT_THUMBSTICK_CLICK_ID, - *RIGHT_THUMBSTICK_TOUCH_ID, - ] - .into_iter() - .collect(), -} -); +pub static HAND_GESTURE_PROFILE_INFO: Lazy = + Lazy::new(|| InteractionProfileInfo { + path: QUEST_CONTROLLER_PROFILE_PATH, + button_set: [ + *LEFT_X_CLICK_ID, + *LEFT_X_TOUCH_ID, + *LEFT_Y_CLICK_ID, + *LEFT_Y_TOUCH_ID, + *LEFT_MENU_CLICK_ID, + *LEFT_SQUEEZE_VALUE_ID, + *LEFT_TRIGGER_VALUE_ID, + *LEFT_TRIGGER_TOUCH_ID, + *LEFT_THUMBSTICK_X_ID, + *LEFT_THUMBSTICK_Y_ID, + *LEFT_THUMBSTICK_CLICK_ID, + *LEFT_THUMBSTICK_TOUCH_ID, + *RIGHT_A_CLICK_ID, + *RIGHT_A_TOUCH_ID, + *RIGHT_B_CLICK_ID, + *RIGHT_B_TOUCH_ID, + *RIGHT_SQUEEZE_VALUE_ID, + *RIGHT_TRIGGER_VALUE_ID, + *RIGHT_TRIGGER_TOUCH_ID, + *RIGHT_THUMBSTICK_X_ID, + *RIGHT_THUMBSTICK_Y_ID, + *RIGHT_THUMBSTICK_CLICK_ID, + *RIGHT_THUMBSTICK_TOUCH_ID, + ] + .into_iter() + .collect(), + }); #[derive(Debug, Clone)] pub struct HandGesture { From f207be829a354a2173234c0b969f788fbee93328 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sun, 10 Sep 2023 11:48:27 +0100 Subject: [PATCH 43/47] add joystick range setting + increase default deadzone --- alvr/server/src/hand_gestures.rs | 6 +++--- alvr/session/src/settings.rs | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 51c6c63771..659e7ce320 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -290,7 +290,7 @@ impl HandGestureManager { }); // Joystick - let joystick_range = 0.01; + let joystick_range = config.joystick_range * 0.01; let joystick_center = lerp_pose(index_intermediate, index_distal, 0.5); let joystick_up = joystick_center @@ -321,10 +321,10 @@ impl HandGestureManager { ); let joystick_contact = index_curl >= 0.75 && grip_curl > 0.5 - && joystick_center.position.distance(thumb_tip.position) <= joystick_range * 5.0 + && joystick_center.position.distance(thumb_tip.position) <= joystick_range * 3.0 && (thumb_tip.position - joystick_center.position).dot(joystick_up) / joystick_up.length() - <= joystick_range * 3.0; + <= joystick_range * 2.0; let joystick_deadzone: f32 = config.joystick_deadzone * 0.01; diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 1cd2d4c147..7784d371c6 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -759,6 +759,13 @@ pub struct HandGestureConfig { #[schema(gui(slider(min = -5.0, max = 5.0)), suffix = "cm")] pub joystick_offset_vertical: f32, + #[schema(flag = "real-time")] + #[schema(strings( + help = "The radius of motion of the joystick. The joystick can be controlled if the thumb is within 2x this range." + ))] + #[schema(gui(slider(min = 0.0, max = 5.0)), suffix = "cm")] + pub joystick_range: f32, + #[schema(flag = "real-time")] #[schema(strings( help = "How long the gesture must be continuously held before it is activated." @@ -1323,9 +1330,10 @@ pub fn session_settings_default() -> SettingsDefault { pinch_trigger_distance: 0.25, curl_touch_distance: 2.0, curl_trigger_distance: 2.5, - joystick_deadzone: 25.0, + joystick_deadzone: 40.0, joystick_offset_horizontal: 0.0, joystick_offset_vertical: 0.0, + joystick_range: 1.0, repeat_delay: 100, activation_delay: 50, deactivation_delay: 100, From de0704aeac35cc538ab66df5659b313b6bfdf191 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:20:13 +0100 Subject: [PATCH 44/47] review response sequel --- alvr/server/src/connection.rs | 71 +++++++++++--------------------- alvr/server/src/hand_gestures.rs | 62 ++++++++++++++-------------- 2 files changed, 55 insertions(+), 78 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 4943a8c93f..9842121988 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -1,7 +1,7 @@ use crate::{ bitrate::BitrateManager, face_tracking::FaceTrackingSink, - hand_gestures::{trigger_hand_gesture_actions, HandGestureManager, HAND_GESTURE_PROFILE_INFO}, + hand_gestures::{trigger_hand_gesture_actions, HandGestureManager, HAND_GESTURE_BUTTON_SET}, haptics, input_mapping::ButtonMappingManager, sockets::WelcomeSocket, @@ -643,37 +643,24 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { thread::spawn(|| ()) }; - let tracking_manager: Arc< - alvr_common::parking_lot::lock_api::Mutex< - alvr_common::parking_lot::RawMutex, - TrackingManager, - >, - > = Arc::new(Mutex::new(TrackingManager::new())); - let hand_gesture_manager: Arc< - alvr_common::parking_lot::lock_api::Mutex< - alvr_common::parking_lot::RawMutex, - HandGestureManager, - >, - > = Arc::new(Mutex::new(HandGestureManager::new())); + let tracking_manager = Arc::new(Mutex::new(TrackingManager::new())); + let hand_gesture_manager = Arc::new(Mutex::new(HandGestureManager::new())); let tracking_receive_thread = thread::spawn({ let tracking_manager = Arc::clone(&tracking_manager); let hand_gesture_manager = Arc::clone(&hand_gesture_manager); - let mut gestures_button_mapping_manager = if let Switch::Enabled(config) = - &SERVER_DATA_MANAGER.read().settings().headset.controllers - { - if let Switch::Enabled(_) = &config.hand_tracking.use_gestures { + let mut gestures_button_mapping_manager = Switch::as_option( + &SERVER_DATA_MANAGER.read().settings().headset.controllers, + ) + .and_then(|config| { + Switch::as_option(&config.hand_tracking.use_gestures).and_then(|_| { Some(ButtonMappingManager::new_automatic( - &HAND_GESTURE_PROFILE_INFO.button_set, + &HAND_GESTURE_BUTTON_SET, &config.button_mapping_config, )) - } else { - None - } - } else { - None - }; + }) + }); move || { let mut face_tracking_sink = @@ -770,8 +757,8 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .into_iter() .map(|(id, motion)| tracking::to_ffi_motion(id, motion)) .collect::>(); - let ffi_left_hand_skeleton = left_hand_skeleton.map(tracking::to_ffi_skeleton); - let ffi_right_hand_skeleton = right_hand_skeleton.map(tracking::to_ffi_skeleton); + let mut ffi_left_hand_skeleton = left_hand_skeleton.map(tracking::to_ffi_skeleton); + let mut ffi_right_hand_skeleton = right_hand_skeleton.map(tracking::to_ffi_skeleton); drop(tracking_manager_lock); @@ -785,23 +772,23 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { { let mut hand_gesture_manager_lock = hand_gesture_manager.lock(); - if tracking.hand_skeletons[0].is_some() { + if let Some(hand_skeleton) = tracking.hand_skeletons[0] { trigger_hand_gesture_actions( gestures_button_mapping_manager.as_mut().unwrap(), *LEFT_HAND_ID, &hand_gesture_manager_lock.get_active_gestures( - tracking.hand_skeletons[0].unwrap(), + hand_skeleton, gestures_config.clone(), *LEFT_HAND_ID, ), ); } - if tracking.hand_skeletons[1].is_some() { + if let Some(hand_skeleton) = tracking.hand_skeletons[1] { trigger_hand_gesture_actions( gestures_button_mapping_manager.as_mut().unwrap(), *RIGHT_HAND_ID, &hand_gesture_manager_lock.get_active_gestures( - tracking.hand_skeletons[1].unwrap(), + hand_skeleton, gestures_config.clone(), *RIGHT_HAND_ID, ), @@ -813,12 +800,12 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { let data_manager_lock = SERVER_DATA_MANAGER.read(); let config = &data_manager_lock.settings().headset; - let hand_skeletons_enabled = - if let Switch::Enabled(controllers) = &config.controllers { - controllers.hand_tracking.enable_skeleton - } else { - false - }; + if let Switch::Enabled(controllers) = &config.controllers { + if !controllers.hand_tracking.enable_skeleton { + ffi_left_hand_skeleton = None; + ffi_right_hand_skeleton = None; + } + } unsafe { crate::SetTracking( @@ -827,20 +814,12 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { ffi_motions.as_ptr(), ffi_motions.len() as _, if let Some(skeleton) = &ffi_left_hand_skeleton { - if hand_skeletons_enabled { - skeleton - } else { - ptr::null() - } + skeleton } else { ptr::null() }, if let Some(skeleton) = &ffi_right_hand_skeleton { - if hand_skeletons_enabled { - skeleton - } else { - ptr::null() - } + skeleton } else { ptr::null() }, diff --git a/alvr/server/src/hand_gestures.rs b/alvr/server/src/hand_gestures.rs index 659e7ce320..5d9a2a4983 100644 --- a/alvr/server/src/hand_gestures.rs +++ b/alvr/server/src/hand_gestures.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, hash::Hash, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -22,37 +22,35 @@ fn lerp_pose(a: Pose, b: Pose, fac: f32) -> Pose { } } -pub static HAND_GESTURE_PROFILE_INFO: Lazy = - Lazy::new(|| InteractionProfileInfo { - path: QUEST_CONTROLLER_PROFILE_PATH, - button_set: [ - *LEFT_X_CLICK_ID, - *LEFT_X_TOUCH_ID, - *LEFT_Y_CLICK_ID, - *LEFT_Y_TOUCH_ID, - *LEFT_MENU_CLICK_ID, - *LEFT_SQUEEZE_VALUE_ID, - *LEFT_TRIGGER_VALUE_ID, - *LEFT_TRIGGER_TOUCH_ID, - *LEFT_THUMBSTICK_X_ID, - *LEFT_THUMBSTICK_Y_ID, - *LEFT_THUMBSTICK_CLICK_ID, - *LEFT_THUMBSTICK_TOUCH_ID, - *RIGHT_A_CLICK_ID, - *RIGHT_A_TOUCH_ID, - *RIGHT_B_CLICK_ID, - *RIGHT_B_TOUCH_ID, - *RIGHT_SQUEEZE_VALUE_ID, - *RIGHT_TRIGGER_VALUE_ID, - *RIGHT_TRIGGER_TOUCH_ID, - *RIGHT_THUMBSTICK_X_ID, - *RIGHT_THUMBSTICK_Y_ID, - *RIGHT_THUMBSTICK_CLICK_ID, - *RIGHT_THUMBSTICK_TOUCH_ID, - ] - .into_iter() - .collect(), - }); +pub static HAND_GESTURE_BUTTON_SET: Lazy> = Lazy::new(|| { + [ + *LEFT_X_CLICK_ID, + *LEFT_X_TOUCH_ID, + *LEFT_Y_CLICK_ID, + *LEFT_Y_TOUCH_ID, + *LEFT_MENU_CLICK_ID, + *LEFT_SQUEEZE_VALUE_ID, + *LEFT_TRIGGER_VALUE_ID, + *LEFT_TRIGGER_TOUCH_ID, + *LEFT_THUMBSTICK_X_ID, + *LEFT_THUMBSTICK_Y_ID, + *LEFT_THUMBSTICK_CLICK_ID, + *LEFT_THUMBSTICK_TOUCH_ID, + *RIGHT_A_CLICK_ID, + *RIGHT_A_TOUCH_ID, + *RIGHT_B_CLICK_ID, + *RIGHT_B_TOUCH_ID, + *RIGHT_SQUEEZE_VALUE_ID, + *RIGHT_TRIGGER_VALUE_ID, + *RIGHT_TRIGGER_TOUCH_ID, + *RIGHT_THUMBSTICK_X_ID, + *RIGHT_THUMBSTICK_Y_ID, + *RIGHT_THUMBSTICK_CLICK_ID, + *RIGHT_THUMBSTICK_TOUCH_ID, + ] + .into_iter() + .collect() +}); #[derive(Debug, Clone)] pub struct HandGesture { From d2c9fc1b21cae6df619e6050351119ec8e91674a Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:38:39 +0100 Subject: [PATCH 45/47] fmt --- alvr/server/src/connection.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 9842121988..719f471551 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -758,7 +758,8 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { .map(|(id, motion)| tracking::to_ffi_motion(id, motion)) .collect::>(); let mut ffi_left_hand_skeleton = left_hand_skeleton.map(tracking::to_ffi_skeleton); - let mut ffi_right_hand_skeleton = right_hand_skeleton.map(tracking::to_ffi_skeleton); + let mut ffi_right_hand_skeleton = + right_hand_skeleton.map(tracking::to_ffi_skeleton); drop(tracking_manager_lock); From 0889776983b55a6aa153fd435d1095ac119195b0 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:40:50 +0100 Subject: [PATCH 46/47] fix collapsible on hand tracking settings --- alvr/session/src/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 7784d371c6..5d130cb25c 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -703,6 +703,7 @@ Currently this cannot be reliably estimated automatically. The correct value sho } #[derive(SettingsSchema, Serialize, Deserialize, Clone)] +#[schema(collapsible)] pub struct HandTrackingConfig { #[schema(flag = "real-time")] #[schema(strings( @@ -1325,7 +1326,6 @@ pub fn session_settings_default() -> SettingsDefault { use_gestures: SwitchDefault { enabled: true, content: HandGestureConfigDefault { - gui_collapsed: true, pinch_touch_distance: 0.0, pinch_trigger_distance: 0.25, curl_touch_distance: 2.0, From 66e5500b37af4f2bdcad2105ae859d0cba91789d Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Mon, 11 Sep 2023 21:48:52 +0100 Subject: [PATCH 47/47] code smell :c --- alvr/server/src/connection.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 719f471551..6ce3a85e85 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -650,17 +650,20 @@ fn try_connect(mut client_ips: HashMap) -> ConResult { let tracking_manager = Arc::clone(&tracking_manager); let hand_gesture_manager = Arc::clone(&hand_gesture_manager); - let mut gestures_button_mapping_manager = Switch::as_option( - &SERVER_DATA_MANAGER.read().settings().headset.controllers, - ) - .and_then(|config| { - Switch::as_option(&config.hand_tracking.use_gestures).and_then(|_| { - Some(ButtonMappingManager::new_automatic( - &HAND_GESTURE_BUTTON_SET, - &config.button_mapping_config, - )) - }) - }); + let mut gestures_button_mapping_manager = SERVER_DATA_MANAGER + .read() + .settings() + .headset + .controllers + .as_option() + .and_then(|config| { + config.hand_tracking.use_gestures.as_option().and_then(|_| { + Some(ButtonMappingManager::new_automatic( + &HAND_GESTURE_BUTTON_SET, + &config.button_mapping_config, + )) + }) + }); move || { let mut face_tracking_sink =