Add short const support, make constant scraping impl. testable

scripts/ChangesSinceLastRelease.txt

@@ -969,6 +969,47 @@ Windows.Win32.NetworkManagement.IpHelper.Apis.SetSessionCompartmentId : return..
 Windows.Win32.NetworkManagement.IpHelper.Apis.SetUnicastIpAddressEntry : return...NTSTATUS => WIN32_ERROR
 # Fixed #1474.
 Windows.Win32.System.ApplicationInstallationAndServicing.ACTIVATION_CONTEXT_COMPATIBILITY_INFORMATION.Elements...Windows.Win32.System.ApplicationInstallationAndServicing.COMPATIBILITY_CONTEXT_ELEMENT* => Windows.Win32.System.ApplicationInstallationAndServicing.COMPATIBILITY_CONTEXT_ELEMENT[]
+# Added support for short PSTR/PWSTR constant values
+Windows.Win32.Media.KernelStreaming.Apis.RT_RCDATA value init changed Int32->Int16
+Windows.Win32.Media.KernelStreaming.Apis.RT_STRING value init changed Int32->Int16
+Windows.Win32.UI.Controls.Apis.TD_ERROR_ICON value init changed Int32->Int16
+Windows.Win32.UI.Controls.Apis.TD_INFORMATION_ICON value init changed Int32->Int16
+Windows.Win32.UI.Controls.Apis.TD_SHIELD_ICON value init changed Int32->Int16
+Windows.Win32.UI.Controls.Apis.TD_WARNING_ICON value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_APPSTARTING value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_ARROW value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_CROSS value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_HAND value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_HELP value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_IBEAM value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_ICON value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_NO value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_PERSON value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_PIN value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZE value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZEALL value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZENESW value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZENS value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZENWSE value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_SIZEWE value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_UPARROW value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.IDC_WAIT value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_ACCELERATOR value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_ANICURSOR value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_ANIICON value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_BITMAP value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_CURSOR value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_DIALOG value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_DLGINCLUDE value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_FONT value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_FONTDIR value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_HTML value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_ICON value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_MENU value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_MESSAGETABLE value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_PLUGPLAY value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_VERSION value init changed Int32->Int16
+Windows.Win32.UI.WindowsAndMessaging.Apis.RT_VXD value init changed Int32->Int16
 # Fixed #1495.
 Windows.Win32.System.ClrHosting.Apis.BucketParamLength added
 Windows.Win32.System.ClrHosting.Apis.BucketParamsCount added

sources/MetadataUtils/ConstantWriter.cs

@@ -5,23 +5,20 @@
 
 namespace MetadataUtils
 {
-    public class ConstantWriter : IDisposable
+    public class ConstantWriter : IDisposable, IConstantWriter
     {
-        private string path;
+        private Stream outputStream;
         private string @namespace;
         private string headerText;
         private StreamWriter writer;
         private Dictionary<string, string> namesToValues = new Dictionary<string, string>();
         private Dictionary<string, string> withAttributes;
 
-        public ConstantWriter(string path, string @namespace, string sourceHeaderText, Dictionary<string, string> withAttributes)
+        public ConstantWriter(Stream outputStream, string @namespace, string sourceHeaderText, Dictionary<string, string> withAttributes)
         {
-            if (sourceHeaderText == null)
-            {
-                sourceHeaderText = string.Empty;
-            }
+            sourceHeaderText ??= string.Empty;
 
-            this.path = path;
+            this.outputStream = outputStream;
             this.@namespace = @namespace;
             this.headerText = sourceHeaderText;
             this.withAttributes = withAttributes;
@@ -247,7 +244,7 @@ private void EnsureStarted()
         {
             if (this.writer == null)
             {
-                this.writer = new StreamWriter(this.path);
+                this.writer = new StreamWriter(this.outputStream);
                 this.writer.WriteLine(
 @$"{this.headerText}
 

sources/MetadataUtils/ConstantsScraper.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.IO.Abstractions;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -76,18 +77,18 @@ private class ConstantsScraperImpl : IDisposable
 
             private static readonly Regex ContainsLowerCase = new Regex(@"[a-z]+");
 
-            private Dictionary<string, EnumWriter> namespacesToEnumWriters = new Dictionary<string, EnumWriter>();
-            private Dictionary<string, ConstantWriter> namespacesToConstantWriters = new Dictionary<string, ConstantWriter>();
+            private Dictionary<string, EnumWriter> namespacesToEnumWriters = new();
+            private Dictionary<string, IConstantWriter> namespacesToConstantWriters = new();
             private WildcardDictionary requiredNamespaces;
             private Dictionary<string, string> scannedNamesToNamespaces;
             private Dictionary<string, string> writtenConstants;
 
-            private List<EnumObject> enumObjectsFromJsons;
+            private List<EnumObject> enumObjectsFromJsons = new();
             private Dictionary<string, string> withTypes;
             private Dictionary<string, string> withAttributes;
 
             private Dictionary<string, List<EnumObject>> enumMemberNameToEnumObj;
-            private HashSet<string> exclusionNames;
+            private HashSet<string> exclusionNames = new();
 
             private string scraperOutputDir;
             private string constantsHeaderText;
@@ -107,7 +108,9 @@ private class ConstantsScraperImpl : IDisposable
                 new RegexConstMaker() { Pattern = @"_WSAIORW\((.+),(.+)\)", ConstType = "uint", OutputFormat = "(IOC_INOUT|({0})|({1}))" },
             };
 
-            private RegexConstHelper regexConstHelper;
+            private IRegexConstHelper regexConstHelper;
+
+            internal IFileSystem _fileSystem { get; set; } = new FileSystem();
 
             public ConstantsScraperImpl()
             {
@@ -211,7 +214,7 @@ private void CleanExistingFiles()
             {
                 foreach (string file in Directory.GetFiles(this.scraperOutputDir).Where(f => f.EndsWith(".enums.cs") || f.EndsWith(".constants.cs")))
                 {
-                    File.Delete(file);
+                    this._fileSystem.File.Delete(file);
                 }
             }
 
@@ -220,12 +223,12 @@ private void InitEnumFlagsFixupFile()
                 if (this.enumFlagsFixupFileName == null)
                 {
                     this.enumFlagsFixupFileName = Path.Combine(this.scraperOutputDir, "enumsMakeFlags.generated.rsp");
-                    if (File.Exists(this.enumFlagsFixupFileName))
+                    if (this._fileSystem.File.Exists(this.enumFlagsFixupFileName))
                     {
-                        File.Delete(this.enumFlagsFixupFileName);
+                        this._fileSystem.File.Delete(this.enumFlagsFixupFileName);
                     }
 
-                    File.AppendAllText(this.enumFlagsFixupFileName, "--enumMakeFlags\r\n");
+                    this._fileSystem.File.AppendAllText(this.enumFlagsFixupFileName, "--enumMakeFlags\r\n");
                 }
             }
 
@@ -384,22 +387,25 @@ private string GetCanonicalGuidConstantIntegerArgs(string args)
                     $"0x{Convert.ToHexString(Encoding.ASCII.GetBytes(m.Groups[1].Value))}", RegexOptions.IgnoreCase);
             }
 
-            private void AddConstantInteger(string originalNamespace, string nativeTypeName, string name, string valueText)
+            private void AddConstantInteger(string originalNamespace, string nativeTypeName, string name, string valueText, string forcedType = "")
             {
                 if (this.writtenConstants.ContainsKey(name))
                 {
                     return;
                 }
 
-                string forcedType = nativeTypeName != null ? null : this.GetForcedTypeForName(name);
+                if (string.IsNullOrWhiteSpace(forcedType))
+                {
+                    forcedType = nativeTypeName != null ? null : this.GetForcedTypeForName(name);
+                }
 
                 var writer = this.GetConstantWriter(originalNamespace, name);
                 writer.AddInt(forcedType, nativeTypeName, name, valueText, out var finalType);
 
                 this.writtenConstants.Add(name, finalType);
             }
 
-            private ConstantWriter GetConstantWriter(string originalNamespace, string name)
+            private IConstantWriter GetConstantWriter(string originalNamespace, string name)
             {
                 string foundNamespace = originalNamespace;
 
@@ -409,15 +415,16 @@ private ConstantWriter GetConstantWriter(string originalNamespace, string name)
                     foundNamespace = newNamespace;
                 }
 
-                if (!this.namespacesToConstantWriters.TryGetValue(foundNamespace, out ConstantWriter constantWriter))
+                if (!this.namespacesToConstantWriters.TryGetValue(foundNamespace, out IConstantWriter constantWriter))
                 {
                     string partConstantsFile = Path.Combine(this.scraperOutputDir, $@"{foundNamespace}.constants.cs");
-                    if (File.Exists(partConstantsFile))
+                    if (this._fileSystem.File.Exists(partConstantsFile))
                     {
-                        File.Delete(partConstantsFile);
+                        this._fileSystem.File.Delete(partConstantsFile);
                     }
 
-                    constantWriter = new ConstantWriter(partConstantsFile, foundNamespace, this.constantsHeaderText, this.withAttributes);
+                    var fileStream = this._fileSystem.File.OpenWrite(partConstantsFile);
+                    constantWriter = new ConstantWriter(fileStream, foundNamespace, this.constantsHeaderText, this.withAttributes);
                     this.namespacesToConstantWriters.Add(foundNamespace, constantWriter);
                 }
 
@@ -426,7 +433,7 @@ private ConstantWriter GetConstantWriter(string originalNamespace, string name)
 
             private HashSet<string> GetManualEnumMemberNames()
             {
-                List<EnumObject> enumObjectsFromManualSources = LoadEnumsFromSourceFiles(Directory.GetFiles(this.scraperOutputDir, "*.manual.cs"));
+                List<EnumObject> enumObjectsFromManualSources = LoadEnumsFromSourceFiles(this._fileSystem.Directory.GetFiles(this.scraperOutputDir, "*.manual.cs"));
                 HashSet<string> manualEnumMemberNames = new HashSet<string>();
                 foreach (EnumObject obj in enumObjectsFromManualSources)
                 {
@@ -465,7 +472,7 @@ private void ScrapeConstantsFromTraversedFiles(Dictionary<string, string> traver
                     var header = item.Key;
                     var currentNamespace = item.Value;
 
-                    if (!File.Exists(header))
+                    if (!this._fileSystem.File.Exists(header))
                     {
                         continue;
                     }
@@ -501,7 +508,7 @@ private void ScrapeConstantsFromTraversedFiles(Dictionary<string, string> traver
                     string defineRegexContinuation = null;
                     bool processingGuidMultiLine = false;
                     string defineGuidKeyword = null;
-                    foreach (string currentLine in File.ReadAllLines(header))
+                    foreach (string currentLine in this._fileSystem.File.ReadAllLines(header))
                     {
                         string fixedCurrentLine = currentLine;
 
@@ -744,7 +751,7 @@ private void ScrapeConstantsFromTraversedFiles(Dictionary<string, string> traver
                                     nativeTypeName = "LPCWSTR";
                                 }
                                 valueText = match.Groups[12].Value;
-                                this.AddConstantInteger(currentNamespace, nativeTypeName, name, valueText);
+                                this.AddConstantInteger(currentNamespace, nativeTypeName, name, valueText, "short");
                                 continue;
                             }
                             // (HWND)-4
@@ -939,7 +946,7 @@ private void TryScrapingEnumFlags(string line)
                     var enumName = match.Groups[1].Value;
 
                     this.InitEnumFlagsFixupFile();
-                    File.AppendAllText(this.enumFlagsFixupFileName, $"{enumName}\r\n");
+                    this._fileSystem.File.AppendAllText(this.enumFlagsFixupFileName, $"{enumName}\r\n");
                 }
             }
 
@@ -1065,9 +1072,9 @@ private void WriteEnumsAndRemaps(
                         {
                             string fixedNamespaceName = foundNamespace.Replace("Windows.Win32.", string.Empty);
                             string enumFile = Path.Combine(this.scraperOutputDir, $"{fixedNamespaceName}.enums.cs");
-                            if (File.Exists(enumFile))
+                            if (this._fileSystem.File.Exists(enumFile))
                             {
-                                File.Delete(enumFile);
+                                this._fileSystem.File.Delete(enumFile);
                             }
 
                             enumWriter = new EnumWriter(enumFile, foundNamespace, this.constantsHeaderText);
@@ -1109,7 +1116,7 @@ private void WriteEnumsAndRemaps(
 
                 if (!linesAdded)
                 {
-                    File.Delete(enumRemapsFileName);
+                    this._fileSystem.File.Delete(enumRemapsFileName);
                 }
 
                 if (this.suggestedEnumRenames.Count != 0)
@@ -1127,7 +1134,7 @@ private class RegexConstMaker
                 public string ConstType { get; set; }
             }
 
-            private class RegexConstHelper
+            private class RegexConstHelper : IRegexConstHelper
             {
                 private ConstantsScraperImpl owner;
                 private List<RegexConstMakerProcessor> processors = new List<RegexConstMakerProcessor>();

sources/MetadataUtils/IConstantWriter.cs

@@ -0,0 +1,11 @@
+namespace MetadataUtils
+{
+    public interface IConstantWriter
+    {
+        void AddGuid(string name, string args);
+        void AddInt(string forceType, string nativeTypeName, string name, string valueText, out string finalType);
+        void AddPropKey(string structType, string name, string args);
+        void AddValue(string type, string name, string valueText, string context = "");
+        void Dispose();
+    }
+}

sources/MetadataUtils/IRegexConstHelper.cs

@@ -0,0 +1,7 @@
+namespace MetadataUtils
+{
+    public interface IRegexConstHelper
+    {
+        bool TryProcessingLine(string originalNamespace, string constName, string value);
+    }
+}

sources/MetadataUtils/MetadataUtils.csproj

@@ -16,6 +16,7 @@
     <PackageReference Include="ICSharpCode.Decompiler" Version="7.2.1.6856" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.3.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="System.IO.Abstractions" Version="19.2.4" />
   </ItemGroup>
 
 </Project>

sources/WinmdUtils/Program.cs

@@ -1139,6 +1139,13 @@ private static bool CompareFields(IField field1, IField field2, DifferencesWrite
                             writer.WriteDifference($"winmd2: {fullField1Name} is a constant with a null value");
                         }
 
+                        var fieldInitType1 = fieldVal1.GetType().Name;
+                        var fieldInitType2 = fieldVal2.GetType().Name;
+                        if (fieldInitType1 != fieldInitType2)
+                        {
+                            writer.WriteDifference($"{fullField1Name} value init changed {fieldInitType1}->{fieldInitType2}");
+                        }
+
                         string val1 = fieldVal1?.ToString();
                         string val2 = fieldVal2?.ToString();