diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/CHANGELOG.md b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/CHANGELOG.md index e69de29..ccbac94 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/CHANGELOG.md +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Hand tracking support for Microsoft HoloLens 2. \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs new file mode 100644 index 0000000..d3d8ca7 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using XRTK.Providers.Controllers.Hands; +using XRTK.WindowsMixedReality.Profiles; + +#if WINDOWS_UWP + +using System; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using Windows.Perception; +using Windows.UI.Input.Spatial; +using XRTK.Definitions.Devices; +using XRTK.Definitions.Utilities; +using XRTK.Services; +using XRTK.Utilities; +using XRTK.WindowsMixedReality.Extensions; +using XRTK.WindowsMixedReality.Utilities; + +#endif // WINDOWS_UWP + +namespace XRTK.WindowsMixedReality.Controllers +{ + /// + /// The Windows Mixed Reality Data Provider for hand controller support. + /// It's responsible for converting the platform data to agnostic data the can work with. + /// + public class WindowsMixedRealityHandControllerDataProvider : BaseHandControllerDataProvider + { + /// + /// Constructor. + /// + /// Name of the data provider as assigned in the configuration profile. + /// Data provider priority controls the order in the service registry. + /// Controller data provider profile assigned to the provider instance in the configuration inspector. + public WindowsMixedRealityHandControllerDataProvider(string name, uint priority, WindowsMixedRealityHandControllerDataProviderProfile profile) + : base(name, priority, profile) + { + } + +#if WINDOWS_UWP + + private readonly WindowsMixedRealityHandDataConverter handDataConverter = new WindowsMixedRealityHandDataConverter(); + private readonly Dictionary activeControllers = new Dictionary(); + + private SpatialInteractionManager spatialInteractionManager = null; + + /// + /// Gets the native instance for the current application + /// state. + /// + private SpatialInteractionManager SpatialInteractionManager + { + get + { + if (spatialInteractionManager == null) + { + UnityEngine.WSA.Application.InvokeOnUIThread(() => + { + spatialInteractionManager = SpatialInteractionManager.GetForCurrentView(); + }, true); + } + + return spatialInteractionManager; + } + } + + #region IMixedRealityControllerDataProvider lifecycle implementation + + /// + public override void Initialize() + { + base.Initialize(); + WindowsMixedRealityHandDataConverter.HandMeshingEnabled = HandMeshingEnabled; + } + + /// + public override void Update() + { + base.Update(); + + // Update existing controllers or create a new one if needed. + var sources = GetCurrentSources(); + + if (sources == null) + { + return; + } + + bool isLeftHandTracked = false; + bool isRightHandTracked = false; + + for (int i = 0; i < sources.Count; i++) + { + var sourceState = sources[i]; + var spatialInteractionSource = sourceState.Source; + + if (spatialInteractionSource.Handedness == SpatialInteractionSourceHandedness.Left) + { + isLeftHandTracked = true; + + if (TryGetController(spatialInteractionSource.Handedness.ToHandedness(), out MixedRealityHandController leftHandController)) + { + leftHandController.UpdateController(handDataConverter.GetHandData(sourceState)); + } + else + { + leftHandController = CreateController(spatialInteractionSource); + leftHandController.UpdateController(handDataConverter.GetHandData(sourceState)); + } + } + + if (spatialInteractionSource.Handedness == SpatialInteractionSourceHandedness.Right) + { + isRightHandTracked = true; + + if (TryGetController(spatialInteractionSource.Handedness.ToHandedness(), out MixedRealityHandController rightHandController)) + { + rightHandController.UpdateController(handDataConverter.GetHandData(sourceState)); + } + else + { + rightHandController = CreateController(spatialInteractionSource); + rightHandController.UpdateController(handDataConverter.GetHandData(sourceState)); + } + } + } + + if (!isLeftHandTracked) + { + RemoveController(Handedness.Left); + } + + if (!isRightHandTracked) + { + RemoveController(Handedness.Right); + } + } + + /// + public override void Disable() + { + foreach (var activeController in activeControllers) + { + RemoveController(activeController.Key, false); + } + + activeControllers.Clear(); + + base.Disable(); + } + + #endregion IMixedRealityControllerDataProvider lifecycle implementation + + #region Controller Management + + /// + /// Reads currently detected input sources by the current instance. + /// + /// List of sources. Can be null. + private IReadOnlyList GetCurrentSources() + { + // Articulated hand support is only present in the 18362 version and beyond Windows + // SDK (which contains the V8 drop of the Universal API Contract). In particular, + // the HandPose related APIs are only present on this version and above. + if (WindowsApiChecker.UniversalApiContractV8_IsAvailable && SpatialInteractionManager != null) + { + var perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now); + var sources = SpatialInteractionManager.GetDetectedSourcesAtTimestamp(perceptionTimestamp); + + if (sources != null) + { + return sources.Where(s => s.Source.Kind == SpatialInteractionSourceKind.Hand).ToList(); + } + } + + return null; + } + + private bool TryGetController(Handedness handedness, out MixedRealityHandController controller) + { + if (activeControllers.ContainsKey(handedness)) + { + var existingController = activeControllers[handedness]; + Debug.Assert(existingController != null, $"Hand Controller {handedness} has been destroyed but remains in the active controller registry."); + controller = existingController; + return true; + } + + controller = null; + return false; + } + + /// + /// Creates the controller for a new device and registers it. + /// + /// Source State provided by the SDK. + /// New controller input source. + private MixedRealityHandController CreateController(SpatialInteractionSource spatialInteractionSource) + { + // We are creating a new controller for the source, determine the type of controller to use. + Type controllerType = spatialInteractionSource.Kind.ToControllerType(); + + if (controllerType == null || controllerType != typeof(MixedRealityHandController)) + { + // This data provider only cares about hands. + return null; + } + + // Ready to create the controller instance. + var controllingHand = spatialInteractionSource.Handedness.ToHandedness(); + var pointers = spatialInteractionSource.IsPointingSupported ? RequestPointers(controllerType, controllingHand, true) : null; + var nameModifier = controllingHand == Handedness.None ? spatialInteractionSource.Kind.ToString() : controllingHand.ToString(); + var inputSource = MixedRealityToolkit.InputSystem?.RequestNewGenericInputSource($"Mixed Reality Hand Controller {nameModifier}", pointers); + var detectedController = new MixedRealityHandController(this, TrackingState.NotApplicable, controllingHand, inputSource); + + if (!detectedController.SetupConfiguration(controllerType)) + { + // Controller failed to be setup correctly. + // Return null so we don't raise the source detected. + return null; + } + + for (int i = 0; i < detectedController.InputSource?.Pointers?.Length; i++) + { + detectedController.InputSource.Pointers[i].Controller = detectedController; + } + + MixedRealityToolkit.InputSystem?.RaiseSourceDetected(detectedController.InputSource, detectedController); + + if (MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.ControllerVisualizationProfile.RenderMotionControllers) + { + detectedController.TryRenderControllerModel(controllerType); + } + + AddController(detectedController); + activeControllers.Add(controllingHand, detectedController); + return detectedController; + } + + private void RemoveController(Handedness handedness, bool removeFromRegistry = true) + { + if (TryGetController(handedness, out var controller)) + { + MixedRealityToolkit.InputSystem?.RaiseSourceLost(controller.InputSource, controller); + + if (removeFromRegistry) + { + RemoveController(controller); + activeControllers.Remove(handedness); + } + } + } + + #endregion Controller Management + +#endif // WINDOWS_UWP + } +} \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs.meta new file mode 100644 index 0000000..5ef8d1f --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsMixedRealityHandControllerDataProvider.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 85d480b696680fb4b9564daac8b82693 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions.meta new file mode 100644 index 0000000..1aceaf6 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b74c6950af5c99c43a77c45eb1062412 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs new file mode 100644 index 0000000..15e261f --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs @@ -0,0 +1,55 @@ +// Copyright (c) XRTK. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#if WINDOWS_UWP + +using Windows.Perception.People; +using XRTK.Definitions.Controllers.Hands; + +namespace XRTK.WindowsMixedReality.Extensions +{ + public static class HandJointKindExtensions + { + public static TrackedHandJoint ToTrackedHandJoint(this HandJointKind handJointKind) + { + switch (handJointKind) + { + case HandJointKind.Palm: return TrackedHandJoint.Palm; + + case HandJointKind.Wrist: return TrackedHandJoint.Wrist; + + case HandJointKind.ThumbMetacarpal: return TrackedHandJoint.ThumbMetacarpalJoint; + case HandJointKind.ThumbProximal: return TrackedHandJoint.ThumbProximalJoint; + case HandJointKind.ThumbDistal: return TrackedHandJoint.ThumbDistalJoint; + case HandJointKind.ThumbTip: return TrackedHandJoint.ThumbTip; + + case HandJointKind.IndexMetacarpal: return TrackedHandJoint.IndexMetacarpal; + case HandJointKind.IndexProximal: return TrackedHandJoint.IndexKnuckle; + case HandJointKind.IndexIntermediate: return TrackedHandJoint.IndexMiddleJoint; + case HandJointKind.IndexDistal: return TrackedHandJoint.IndexDistalJoint; + case HandJointKind.IndexTip: return TrackedHandJoint.IndexTip; + + case HandJointKind.MiddleMetacarpal: return TrackedHandJoint.MiddleMetacarpal; + case HandJointKind.MiddleProximal: return TrackedHandJoint.MiddleKnuckle; + case HandJointKind.MiddleIntermediate: return TrackedHandJoint.MiddleMiddleJoint; + case HandJointKind.MiddleDistal: return TrackedHandJoint.MiddleDistalJoint; + case HandJointKind.MiddleTip: return TrackedHandJoint.MiddleTip; + + case HandJointKind.RingMetacarpal: return TrackedHandJoint.RingMetacarpal; + case HandJointKind.RingProximal: return TrackedHandJoint.RingKnuckle; + case HandJointKind.RingIntermediate: return TrackedHandJoint.RingMiddleJoint; + case HandJointKind.RingDistal: return TrackedHandJoint.RingDistalJoint; + case HandJointKind.RingTip: return TrackedHandJoint.RingTip; + + case HandJointKind.LittleMetacarpal: return TrackedHandJoint.PinkyMetacarpal; + case HandJointKind.LittleProximal: return TrackedHandJoint.PinkyKnuckle; + case HandJointKind.LittleIntermediate: return TrackedHandJoint.PinkyMiddleJoint; + case HandJointKind.LittleDistal: return TrackedHandJoint.PinkyDistalJoint; + case HandJointKind.LittleTip: return TrackedHandJoint.PinkyTip; + + default: return TrackedHandJoint.None; + } + } + } +} +#endif // WINDOWS_UWP \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs.meta new file mode 100644 index 0000000..707b6bd --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/HandJointKindExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ed178a03328a32040bdc40c672fa4639 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs new file mode 100644 index 0000000..c228204 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs @@ -0,0 +1,33 @@ +// Copyright (c) XRTK. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#if WINDOWS_UWP + +using Windows.UI.Input.Spatial; +using XRTK.Definitions.Utilities; + +namespace XRTK.WindowsMixedReality.Extensions +{ + public static class SpatialInteractionSourceHandednessExtensions + { + /// + /// Converts the native to a XRTK . + /// + /// Value to convert. + /// The XRTK value. + public static Handedness ToHandedness(this SpatialInteractionSourceHandedness spatialInteractionSourceHandedness) + { + switch (spatialInteractionSourceHandedness) + { + case SpatialInteractionSourceHandedness.Left: + return Handedness.Left; + case SpatialInteractionSourceHandedness.Right: + return Handedness.Right; + default: + return Handedness.None; + } + } + + } +} +#endif // WINDOWS_UWP \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs.meta new file mode 100644 index 0000000..aa37f8d --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceHandednessExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e18c9e0f33e27424490d7c5a5ef2eda6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs new file mode 100644 index 0000000..08c1c4f --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs @@ -0,0 +1,35 @@ +// Copyright (c) XRTK. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#if WINDOWS_UWP + +using System; +using Windows.UI.Input.Spatial; +using XRTK.Providers.Controllers.Hands; +using XRTK.WindowsMixedReality.Controllers; + +namespace XRTK.WindowsMixedReality.Extensions +{ + public static class SpatialInteractionSourceKindExtensions + { + /// + /// Maps the native to a XRTK type. + /// + /// Value to map. + /// The XRTK type representing the source kind. + public static Type ToControllerType(this SpatialInteractionSourceKind spatialInteractionSourceKind) + { + switch (spatialInteractionSourceKind) + { + case SpatialInteractionSourceKind.Controller: + return typeof(WindowsMixedRealityController); + case SpatialInteractionSourceKind.Hand: + return typeof(MixedRealityHandController); + default: + return null; + } + } + + } +} +#endif // WINDOWS_UWP \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs.meta new file mode 100644 index 0000000..a7db3a7 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Extensions/SpatialInteractionSourceKindExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: adc455969459d0c4bad79c183d6c1eb4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityDataProviderProfileInspector.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityControllerDataProviderProfileInspector.cs similarity index 95% rename from XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityDataProviderProfileInspector.cs rename to XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityControllerDataProviderProfileInspector.cs index 1b6b81a..75f4f9f 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityDataProviderProfileInspector.cs +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityControllerDataProviderProfileInspector.cs @@ -8,7 +8,7 @@ namespace XRTK.WindowsMixedReality.Inspectors { [CustomEditor(typeof(WindowsMixedRealityControllerDataProviderProfile))] - public class WindowsMixedRealityDataProviderProfileInspector : BaseMixedRealityProfileInspector + public class WindowsMixedRealityControllerDataProviderProfileInspector : BaseMixedRealityProfileInspector { private SerializedProperty manipulationGestures; private SerializedProperty useRailsNavigation; diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityDataProviderProfileInspector.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityControllerDataProviderProfileInspector.cs.meta similarity index 100% rename from XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityDataProviderProfileInspector.cs.meta rename to XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityControllerDataProviderProfileInspector.cs.meta diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs new file mode 100644 index 0000000..b99814d --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs @@ -0,0 +1,23 @@ +// Copyright (c) XRTK. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using UnityEditor; +using XRTK.Inspectors.Profiles.InputSystem.Controllers; +using XRTK.WindowsMixedReality.Profiles; + +namespace XRTK.WindowsMixedReality.Inspectors +{ + [CustomEditor(typeof(WindowsMixedRealityHandControllerDataProviderProfile))] + public class WindowsMixedRealityHandControllerDataProviderProfileInspector : BaseMixedRealityHandControllerDataProviderProfileInspector + { + public override void OnInspectorGUI() + { + RenderHeader(); + + EditorGUILayout.Space(); + EditorGUILayout.LabelField("Windows Mixed Reality Hand Controller Data Provider Settings", EditorStyles.boldLabel); + + base.OnInspectorGUI(); + } + } +} \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs.meta new file mode 100644 index 0000000..c7b976e --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Inspectors/WindowsMixedRealityHandControllerDataProviderProfileInspector.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: fd86308f7d99ef344bf0b3259392f28a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 6e2e9d716bbb4d8382bd53f11996b90e, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/LICENSE.md b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/LICENSE.md index 9ba57df..ae38a2f 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/LICENSE.md +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 XRTK +Copyright (c) 2020 XRTK Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityControllerDataProviderProfile.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityControllerDataProviderProfile.cs index 4a9e39e..4be8456 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityControllerDataProviderProfile.cs +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityControllerDataProviderProfile.cs @@ -50,4 +50,4 @@ public class WindowsMixedRealityControllerDataProviderProfile : BaseMixedReality public AutoStartBehavior WindowsGestureAutoStart => windowsGestureAutoStart; } -} +} \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs new file mode 100644 index 0000000..b1e7ffb --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs @@ -0,0 +1,14 @@ +// Copyright (c) XRTK. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using UnityEngine; +using XRTK.Definitions.Utilities; +using XRTK.Definitions.Controllers.Hands; + +namespace XRTK.WindowsMixedReality.Profiles +{ + [CreateAssetMenu(menuName = "Mixed Reality Toolkit/Input System/Controller Data Providers/Windows Mixed Reality Hand", fileName = "WindowsMixedRealityHandControllerDataProviderProfile", order = (int)CreateProfileMenuItemIndices.Input)] + public class WindowsMixedRealityHandControllerDataProviderProfile : BaseHandControllerDataProviderProfile + { + } +} \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs.meta new file mode 100644 index 0000000..28de0ee --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Profiles/WindowsMixedRealityHandControllerDataProviderProfile.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 29a54eda3f140e7418c5cb5c2bccd329 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 6e2e9d716bbb4d8382bd53f11996b90e, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/README.md b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/README.md index d0fb283..7b8074d 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/README.md +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/README.md @@ -1,4 +1,5 @@ # WindowsMixedReality + The Windows Mixed Reality Extension for the [XRTK - Mixed Reality Toolkit](https://github.com/XRTK/XRTK-Core) ## Build Status diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities.meta new file mode 100644 index 0000000..56bc6a8 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: ad33f0fa1dc0f344cb7c818af4cc7908 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs new file mode 100644 index 0000000..2b0817e --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#if WINDOWS_UWP + +using System; +using System.Collections.Generic; +using UnityEngine; +using Windows.Perception.People; +using Windows.UI.Input.Spatial; +using XRTK.Definitions.Controllers.Hands; +using XRTK.Definitions.Utilities; +using XRTK.Extensions; +using XRTK.Services; +using XRTK.WindowsMixedReality.Extensions; + +namespace XRTK.WindowsMixedReality.Utilities +{ + /// + /// Converts windows mixed reality hand data to . + /// + public sealed class WindowsMixedRealityHandDataConverter + { + /// + /// Gets or sets whether hand mesh data should be read and converted. + /// + public static bool HandMeshingEnabled { get; set; } + + private readonly Vector3[] unityJointPositions = new Vector3[jointIndices.Length]; + private readonly Quaternion[] unityJointOrientations = new Quaternion[jointIndices.Length]; + private readonly Dictionary handMeshObservers = new Dictionary(); + + private int[] handMeshTriangleIndices = null; + private bool hasRequestedHandMeshObserverLeftHand = false; + private bool hasRequestedHandMeshObserverRightHand = false; + private Vector2[] handMeshUVs; + + private static readonly HandJointKind[] jointIndices = new HandJointKind[] + { + HandJointKind.Palm, + HandJointKind.Wrist, + HandJointKind.ThumbMetacarpal, + HandJointKind.ThumbProximal, + HandJointKind.ThumbDistal, + HandJointKind.ThumbTip, + HandJointKind.IndexMetacarpal, + HandJointKind.IndexProximal, + HandJointKind.IndexIntermediate, + HandJointKind.IndexDistal, + HandJointKind.IndexTip, + HandJointKind.MiddleMetacarpal, + HandJointKind.MiddleProximal, + HandJointKind.MiddleIntermediate, + HandJointKind.MiddleDistal, + HandJointKind.MiddleTip, + HandJointKind.RingMetacarpal, + HandJointKind.RingProximal, + HandJointKind.RingIntermediate, + HandJointKind.RingDistal, + HandJointKind.RingTip, + HandJointKind.LittleMetacarpal, + HandJointKind.LittleProximal, + HandJointKind.LittleIntermediate, + HandJointKind.LittleDistal, + HandJointKind.LittleTip + }; + + /// + /// Gets updated hand data for the current frame. + /// + /// Platform provided current input source state for the hand. + /// Platform agnostics hand data. + public HandData GetHandData(SpatialInteractionSourceState spatialInteractionSourceState) + { + HandPose handPose = spatialInteractionSourceState.TryGetHandPose(); + HandData updatedHandData = new HandData + { + IsTracked = handPose != null, + TimeStamp = DateTimeOffset.UtcNow.Ticks + }; + + if (updatedHandData.IsTracked) + { + // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it. + if (HandMeshingEnabled) + { + if (!handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness) && !HasRequestedHandMeshObserver(spatialInteractionSourceState.Source.Handedness)) + { + SetHandMeshObserver(spatialInteractionSourceState); + } + + if (handMeshObservers.TryGetValue(spatialInteractionSourceState.Source.Handedness, out var handMeshObserver) && handMeshTriangleIndices == null) + { + uint indexCount = handMeshObserver.TriangleIndexCount; + ushort[] indices = new ushort[indexCount]; + handMeshObserver.GetTriangleIndices(indices); + handMeshTriangleIndices = new int[indexCount]; + Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount); + + // Compute neutral pose + Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount]; + HandPose neutralPose = handMeshObserver.NeutralPose; + var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; + HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose); + handMeshVertexState.GetVertices(vertexAndNormals); + + for (int i = 0; i < handMeshObserver.VertexCount; i++) + { + neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnity(); + } + + // Compute UV mapping + InitializeHandMeshUVs(neutralPoseVertices); + } + + if (handMeshObserver != null && handMeshTriangleIndices != null) + { + var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; + var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose); + handMeshVertexState.GetVertices(vertexAndNormals); + + var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem); + if (meshTransform.HasValue) + { + System.Numerics.Vector3 scale; + System.Numerics.Quaternion rotation; + System.Numerics.Vector3 translation; + System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation); + + var handMeshVertices = new Vector3[handMeshObserver.VertexCount]; + var handMeshNormals = new Vector3[handMeshObserver.VertexCount]; + + for (int i = 0; i < handMeshObserver.VertexCount; i++) + { + handMeshVertices[i] = vertexAndNormals[i].Position.ToUnity(); + handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnity(); + } + + updatedHandData.Mesh = new HandMeshData( + handMeshVertices, + handMeshTriangleIndices, + handMeshNormals, + handMeshUVs, + translation.ToUnity(), + rotation.ToUnity()); + } + } + } + else if (handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness)) + { + // if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created + if (spatialInteractionSourceState.Source.Handedness == SpatialInteractionSourceHandedness.Left) + { + hasRequestedHandMeshObserverLeftHand = false; + } + else if (spatialInteractionSourceState.Source.Handedness == SpatialInteractionSourceHandedness.Right) + { + hasRequestedHandMeshObserverRightHand = false; + } + + handMeshObservers.Remove(spatialInteractionSourceState.Source.Handedness); + } + + JointPose[] jointPoses = new JointPose[jointIndices.Length]; + if (handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses)) + { + for (int i = 0; i < jointPoses.Length; i++) + { + unityJointOrientations[i] = jointPoses[i].Orientation.ToUnity(); + unityJointPositions[i] = jointPoses[i].Position.ToUnity(); + + // We want the controller to follow the Playspace, so fold in the playspace transform here to + // put the controller pose into world space. + unityJointPositions[i] = MixedRealityToolkit.CameraSystem.CameraRig.PlayspaceTransform.TransformPoint(unityJointPositions[i]); + unityJointOrientations[i] = MixedRealityToolkit.CameraSystem.CameraRig.PlayspaceTransform.rotation * unityJointOrientations[i]; + + TrackedHandJoint handJoint = jointIndices[i].ToTrackedHandJoint(); + updatedHandData.Joints[(int)handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]); + } + } + } + + return updatedHandData; + } + + private void InitializeHandMeshUVs(Vector3[] neutralPoseVertices) + { + if (neutralPoseVertices.Length == 0) + { + Debug.LogError("Loaded 0 verts for neutralPoseVertices"); + } + + float minY = neutralPoseVertices[0].y; + float maxY = minY; + + float maxMagnitude = 0.0f; + + for (int ix = 1; ix < neutralPoseVertices.Length; ix++) + { + Vector3 p = neutralPoseVertices[ix]; + + if (p.y < minY) + { + minY = p.y; + } + else if (p.y > maxY) + { + maxY = p.y; + } + float d = p.x * p.x + p.y * p.y; + if (d > maxMagnitude) maxMagnitude = d; + } + + maxMagnitude = Mathf.Sqrt(maxMagnitude); + float scale = 1.0f / (maxY - minY); + + handMeshUVs = new Vector2[neutralPoseVertices.Length]; + + for (int ix = 0; ix < neutralPoseVertices.Length; ix++) + { + Vector3 p = neutralPoseVertices[ix]; + + handMeshUVs[ix] = new Vector2(p.x * scale + 0.5f, (p.y - minY) * scale); + } + } + + private async void SetHandMeshObserver(SpatialInteractionSourceState sourceState) + { + if (handMeshObservers.ContainsKey(sourceState.Source.Handedness)) + { + handMeshObservers[sourceState.Source.Handedness] = await sourceState.Source.TryCreateHandMeshObserverAsync(); + } + else + { + handMeshObservers.Add(sourceState.Source.Handedness, await sourceState.Source.TryCreateHandMeshObserverAsync()); + } + + hasRequestedHandMeshObserverLeftHand = sourceState.Source.Handedness == SpatialInteractionSourceHandedness.Left; + hasRequestedHandMeshObserverRightHand = sourceState.Source.Handedness == SpatialInteractionSourceHandedness.Right; + } + + private bool HasRequestedHandMeshObserver(SpatialInteractionSourceHandedness handedness) + { + return handedness == SpatialInteractionSourceHandedness.Left + ? hasRequestedHandMeshObserverLeftHand + : handedness == SpatialInteractionSourceHandedness.Right && hasRequestedHandMeshObserverRightHand; + } + + } +} +#endif // WINDOWS_UWP \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs.meta new file mode 100644 index 0000000..c94c07a --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityHandDataConverter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 25d284a3f798cfa4bbe5a4c86eb9527b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs new file mode 100644 index 0000000..0b6e8a5 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +#if WINDOWS_UWP + +using System.Runtime.InteropServices; +using UnityEngine.XR.WSA; +using Windows.Perception.Spatial; + +namespace XRTK.WindowsMixedReality.Utilities +{ + public static class WindowsMixedRealityUtilities + { + private static SpatialCoordinateSystem spatialCoordinateSystem = null; + + public static SpatialCoordinateSystem SpatialCoordinateSystem => spatialCoordinateSystem ?? (spatialCoordinateSystem = Marshal.GetObjectForIUnknown(WorldManager.GetNativeISpatialCoordinateSystemPtr()) as SpatialCoordinateSystem); + } +} +#endif // WINDOWS_UWP \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs.meta new file mode 100644 index 0000000..95dd391 --- /dev/null +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Utilities/WindowsMixedRealityUtilities.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ac87f6ccddeb0f347b434c702ff52189 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json index a3cc9b4..69b820a 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json @@ -1,7 +1,7 @@ { "name": "com.xrtk.wmr", "displayName": "XRTK.WindowsMixedReality", - "description": "The Windows Mixed Reality components for the Mixed Reality Toolkit", + "description": "The Windows Mixed Reality components for the Mixed Reality Toolkit.", "keywords": [ "xrtk", "vr", @@ -10,12 +10,12 @@ "mixed", "reality" ], - "version": "0.1.13", + "version": "0.2.0", "unity": "2019.1", "license": "MIT", "author": "XRTK Team (https://github.com/XRTK)", "dependencies": { - "com.xrtk.core": "0.1.33", + "com.xrtk.core": "0.2.0", "com.unity.xr.windowsmr.metro": "1.0.19" } } \ No newline at end of file