feat: Change to allow MouthMorph to handle other than the main face mesh

Packages/jp.suzuryg.face-emo/Editor/Detail/AV3/FxGenerator.cs

@@ -1254,23 +1254,44 @@ private AacFlClip GetMouthMorphCancelerAnimation(AV3Setting aV3Setting, AacFlBas
         {
             var clip = aac.NewClip();
 
-            var faceMesh = AV3Utility.GetFaceMesh(avatarDescriptor);
-            if (faceMesh == null)
+            #pragma warning disable CS0612
+            var obsolete = aV3Setting.MouthMorphBlendShapes;
+            if (obsolete.Any())
             {
-                return clip;
+                var faceMesh = AV3Utility.GetFaceMesh(avatarDescriptor);
+                if (faceMesh != null)
+                {
+                    var faceMeshPath = AV3Utility.GetPathFromAvatarRoot(faceMesh.transform, avatarDescriptor);
+                    foreach (var name in obsolete)
+                    {
+                        var blendShape = new BlendShape(path: faceMeshPath, name: name);
+                        if (!aV3Setting.MouthMorphs.Contains(blendShape)) { aV3Setting.MouthMorphs.Add(blendShape); }
+                    }
+                    obsolete.Clear();
+                    EditorUtility.SetDirty(aV3Setting);
+                }
             }
+            #pragma warning restore CS0612
 
-            // Generate clip
-            var mouthMorphBlendShapes = new HashSet<string>(aV3Setting.MouthMorphBlendShapes);
-            var excludeBlink = !_aV3Setting.ReplaceBlink; // If blinking is not replaced by animation, do not reset the shape key for blinking
+            var excludeBlink = false;
             var excludeLipSync = true;
             var blendShapeValues = AV3Utility.GetFaceMeshBlendShapeValues(avatarDescriptor, excludeBlink, excludeLipSync);
+            foreach (var mesh in aV3Setting.AdditionalSkinnedMeshes)
+            {
+                var blendShapes = AV3Utility.GetBlendShapeValues(mesh, avatarDescriptor, excludeBlink, excludeLipSync);
+                foreach (var item in blendShapes) { blendShapeValues[item.Key] = item.Value; }
+            }
+
+            var mouthMorphBlendShapes = new HashSet<BlendShape>(aV3Setting.MouthMorphs);
+
+            var cachedMeshes = new Dictionary<string, SkinnedMeshRenderer>();
             foreach (var blendShape in blendShapeValues.Keys)
             {
                 var weight = blendShapeValues[blendShape];
-                if (mouthMorphBlendShapes.Contains(blendShape.Name))
+                if (mouthMorphBlendShapes.Contains(blendShape))
                 {
-                    clip = clip.BlendShape(faceMesh, blendShape.Name, weight);
+                    var mesh = cachedMeshes.ContainsKey(blendShape.Path) ? cachedMeshes[blendShape.Path] : AV3Utility.GetMeshByPath(blendShape.Path, avatarDescriptor);
+                    clip = clip.BlendShape(mesh, blendShape.Name, weight);
                 }
             }
 

Packages/jp.suzuryg.face-emo/Editor/Detail/View/InspectorView.cs

@@ -15,6 +15,8 @@
 using UnityEditorInternal;
 using UniRx;
 using VRC.SDK3.Avatars.Components;
+using UnityEditor.SceneManagement;
+using UnityEngine.SceneManagement;
 
 namespace Suzuryg.FaceEmo.Detail.View
 {
@@ -113,7 +115,7 @@ public InspectorView(
             };
 
             // Mouth morph blendshapes
-            _mouthMorphBlendShapes = new ReorderableList(null, typeof(string));
+            _mouthMorphBlendShapes = new ReorderableList(null, typeof(BlendShape));
             _mouthMorphBlendShapes.onAddCallback = AddMouthMorphBlendShape;
             _mouthMorphBlendShapes.onRemoveCallback = RemoveMouthMorphBlendShape;
             _mouthMorphBlendShapes.draggable = false;
@@ -421,50 +423,39 @@ private void Field_ApplyingToMultipleAvatars()
             }
         }
 
-        private List<string> GetMouthMorphBlendShapes()
-        {
-            _av3Setting.Update();
-            var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphBlendShapes));
-
-            var list = new List<string>();
-            for (int i = 0; i < property.arraySize; i++)
-            {
-                list.Add(property.GetArrayElementAtIndex(i).stringValue);
-            }
-
-            return list;
-        }
-
         private void AddMouthMorphBlendShape(ReorderableList reorderableList)
         {
-            var faceBlendShapes = new List<string>();
+            var faceBlendShapes = new List<BlendShape>();
             _av3Setting.Update();
             var avatarDescriptor = _av3Setting.FindProperty(nameof(AV3Setting.TargetAvatar)).objectReferenceValue as VRCAvatarDescriptor;
             if (avatarDescriptor != null)
             {
                 var replaceBlink = _av3Setting.FindProperty(nameof(AV3Setting.ReplaceBlink)).boolValue;
-                var excludeBlink = !replaceBlink; // If blinking is not replaced by animation, do not reset the shape key for blinking
+                var excludeBlink = false;
                 var excludeLipSync = true;
-                faceBlendShapes = AV3Utility.GetFaceMeshBlendShapeValues(avatarDescriptor, excludeBlink, excludeLipSync).Select(x => x.Key.Name).ToList();
+                faceBlendShapes = AV3Utility.GetFaceMeshBlendShapeValues(avatarDescriptor, excludeBlink, excludeLipSync).Select(x => x.Key).ToList();
+
+                foreach (var mesh in GetValue<List<SkinnedMeshRenderer>>(_av3Setting.FindProperty(nameof(AV3Setting.AdditionalSkinnedMeshes))))
+                {
+                    var blendShapes = AV3Utility.GetBlendShapeValues(mesh, avatarDescriptor, excludeBlink, excludeLipSync);
+                    foreach (var item in blendShapes) { faceBlendShapes.Add(item.Key); }
+                }
             }
 
             UnityEditor.PopupWindow.Show(
                 new Rect(Event.current.mousePosition, Vector2.one),
-                new ListSelectPopupContent<string>(faceBlendShapes, _localizationTable.Common_Add, _localizationTable.Common_Cancel,
-                new Action<IReadOnlyList<string>>(blendShapes =>
+                new ListSelectPopupContent<BlendShape>(faceBlendShapes, _localizationTable.Common_Add, _localizationTable.Common_Cancel,
+                new Action<IReadOnlyList<BlendShape>>(added =>
                 {
-                    var existing = new HashSet<string>(GetMouthMorphBlendShapes());
-                    var toAdd = blendShapes.Distinct().Where(x => !existing.Contains(x)).ToList();
-
                     _av3Setting.Update();
 
-                    var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphBlendShapes));
-                    var start = property.arraySize;
-                    property.arraySize += toAdd.Count;
-                    for (int i = 0; i < toAdd.Count; i++)
+                    var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphs));
+                    var list = GetValue<List<BlendShape>>(property);
+                    foreach (var item in added)
                     {
-                        property.GetArrayElementAtIndex(i + start).stringValue = toAdd[i];
+                        if (!list.Contains(item)) { list.Add(item); }
                     }
+                    SetValue(property, list);
 
                     _av3Setting.ApplyModifiedProperties();
                 })));
@@ -478,20 +469,17 @@ private void RemoveMouthMorphBlendShape(ReorderableList reorderableList)
                 return;
             }
 
-            var selected = _mouthMorphBlendShapes.list[_mouthMorphBlendShapes.index] as string;
+            var selected = _mouthMorphBlendShapes.list[_mouthMorphBlendShapes.index] as BlendShape;
 
             _av3Setting.Update();
 
-            var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphBlendShapes));
-            for (int i = 0; i < property.arraySize; i++)
+            var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphs));
+            var list = GetValue<List<BlendShape>>(property);
+            while (list.Contains(selected))
             {
-                var element = property.GetArrayElementAtIndex(i);
-                if (element.stringValue == selected)
-                {
-                    // Remove all duplicate elements without breaking to be sure.
-                    property.DeleteArrayElementAtIndex(i);
-                }
+                list.Remove(selected);
             }
+            SetValue(property, list);
 
             _av3Setting.ApplyModifiedProperties();
         }
@@ -506,7 +494,33 @@ private void Field_MouthMorphBlendShape()
                 HelpBoxDrawer.InfoLayout(_localizationTable.InspectorView_Tooltip_ConfirmMouthMorphBlendShape);
             }
 
-            _mouthMorphBlendShapes.list = GetMouthMorphBlendShapes(); // Is it necessary to get every frame?
+            _av3Setting.Update();
+            var mouthMorphsProperty = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphs));
+            var mouthMorphs = GetValue<List<BlendShape>>(mouthMorphsProperty);
+
+            #pragma warning disable CS0612
+            var obsoleteProperty = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphBlendShapes));
+            if (obsoleteProperty.arraySize > 0)
+            {
+                var obsolete = GetValue<List<string>>(obsoleteProperty);
+                var avatarDescriptor = _av3Setting.FindProperty(nameof(AV3Setting.TargetAvatar)).objectReferenceValue as VRCAvatarDescriptor;
+                var faceMesh = AV3Utility.GetFaceMesh(avatarDescriptor);
+                if (faceMesh != null)
+                {
+                    var faceMeshPath = AV3Utility.GetPathFromAvatarRoot(faceMesh.transform, avatarDescriptor);
+                    foreach (var name in obsolete)
+                    {
+                        var blendShape = new BlendShape(path: faceMeshPath, name: name);
+                        if (!mouthMorphs.Contains(blendShape)) { mouthMorphs.Add(blendShape); }
+                    }
+                    obsolete.Clear();
+                    SetValue(obsoleteProperty, obsolete);
+                    SetValue(mouthMorphsProperty, mouthMorphs);
+                }
+            }
+            #pragma warning restore CS0612
+
+            _mouthMorphBlendShapes.list = mouthMorphs; // Is it necessary to get every frame?
             _mouthMorphBlendShapes.DoLayoutList();
 
             EditorGUILayout.Space(10);
@@ -516,8 +530,8 @@ private void Field_MouthMorphBlendShape()
                     _localizationTable.InspectorView_Message_ClearMouthMorphBlendShapes,
                     _localizationTable.Common_Delete, _localizationTable.Common_Cancel, isRiskyAction: true))
             {
-                var property = _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphBlendShapes));
-                property.ClearArray();
+                _av3Setting.FindProperty(nameof(AV3Setting.MouthMorphs)).ClearArray();
+                _av3Setting.ApplyModifiedProperties();
             }
         }
 
@@ -894,5 +908,27 @@ bool IsDescendantOf(Transform child, Transform potentialAncestor)
             }
             return false;
         }
+
+        private static T GetValue<T>(SerializedProperty property)
+        {
+            object obj = property.serializedObject.targetObject;
+            System.Reflection.FieldInfo fieldInfo = obj.GetType().GetField(property.propertyPath);
+            if (fieldInfo != null)
+            {
+                return (T)fieldInfo.GetValue(obj);
+            }
+            return default(T);
+        }
+
+        private static void SetValue<T>(SerializedProperty property, T newValue)
+        {
+            object obj = property.serializedObject.targetObject;
+            System.Reflection.FieldInfo fieldInfo = obj.GetType().GetField(property.propertyPath);
+            if (fieldInfo != null)
+            {
+                fieldInfo.SetValue(obj, newValue);
+            }
+            EditorSceneManager.MarkSceneDirty(SceneManager.GetActiveScene());
+        }
     }
 }

Packages/jp.suzuryg.face-emo/Runtime/Components/Settings/AV3Setting.cs

@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using Suzuryg.FaceEmo.Domain;
+using System;
+using System.Collections.Generic;
 using UnityEngine;
 
 namespace Suzuryg.FaceEmo.Components.Settings
@@ -12,7 +14,10 @@ public class AV3Setting : ScriptableObject
 
         public GameObject MARootObjectPrefab;
 
+        [Obsolete]
         public List<string> MouthMorphBlendShapes = new List<string>();
+        public List<BlendShape> MouthMorphs = new List<BlendShape>();
+
         public List<SkinnedMeshRenderer> AdditionalSkinnedMeshes = new List<SkinnedMeshRenderer>();
         public List<GameObject> AdditionalToggleObjects = new List<GameObject>();
         public List<GameObject> AdditionalTransformObjects = new List<GameObject>();

Packages/jp.suzuryg.face-emo/Editor/Detail/BlendShape.cs → Packages/jp.suzuryg.face-emo/Runtime/Domain/BlendShape.cs

@@ -1,7 +1,7 @@
 using System;
 using UnityEngine;
 
-namespace Suzuryg.FaceEmo.Detail
+namespace Suzuryg.FaceEmo.Domain
 {
     [Serializable]
     public class BlendShape : IEquatable<BlendShape>
@@ -25,6 +25,11 @@ public BlendShape(string path, string name)
             _name = name;
         }
 
+        public override string ToString()
+        {
+            return _path + "." + _name;
+        }
+
         public bool Equals(BlendShape other)
         {
             if (other is null)

Packages/jp.suzuryg.face-emo/Runtime/Domain/jp.suzuryg.face-emo.domain.Runtime.asmdef

@@ -9,5 +9,5 @@
     "autoReferenced": false,
     "defineConstraints": [],
     "versionDefines": [],
-    "noEngineReferences": true
+    "noEngineReferences": false
 }

Packages/jp.suzuryg.face-emo/Tests/Editor/Detail/BlendShapeTests.cs

@@ -1,4 +1,5 @@
 using NUnit.Framework;
+using Suzuryg.FaceEmo.Domain;
 using System.Collections.Generic;
 using System.Linq;
 using UnityEngine;