Implement parsing OpenAPI to internal model (opensearch-project#228)

Signed-off-by: Thomas Farr <tsfarr@amazon.com>

src/ApiGenerator/ApiGenerator.csproj

@@ -11,8 +11,8 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.2.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="NSwag.Core.Yaml" Version="13.19.0" />
     <PackageReference Include="ShellProgressBar" Version="5.2.0" />
-    <PackageReference Include="CsQuery.Core" Version="2.0.1" />
     <PackageReference Include="Spectre.Console" Version="0.47.0" />
     <PackageReference Include="System.CommandLine.DragonFruit" Version="0.3.0-alpha.20371.2" />
     <PackageReference Include="RazorLight" Version="2.1.0" />

src/ApiGenerator/Configuration/CodeConfiguration.cs

@@ -36,37 +36,7 @@ namespace ApiGenerator.Configuration
 {
     public static class CodeConfiguration
     {
-        /// <summary> These APIs are not implemented yet in the low and high level client</summary>
-        public static string[] IgnoredApis { get; } =
-        {
-            // To be removed
-            "indices.upgrade.json",
-            "indices.get_upgrade.json",
-        };
-
-        private static string[] IgnoredApisHighLevel { get; } =
-        {
-            "indices.delete_index_template.json",
-            "indices.exists_index_template.json",
-            "indices.get_index_template.json",
-            "indices.put_index_template.json",
-            "indices.simulate_index_template.json",
-            "indices.simulate_template.json",
-
-            "get_script_context.json", // 7.7 experimental
-            "get_script_languages.json", // 7.7 experimental
-
-            "indices.exist_type.json", // already removed on client
-
-            "rank_eval.json", // 7.7 experimental
-            "scripts_painless_context.json", // 7.7 experimental
-            "cluster.delete_component_template.json", // 7.8 experimental
-            "cluster.get_component_template.json", // 7.8 experimental
-            "cluster.put_component_template.json", // 7.8 experimental
-            "cluster.exists_component_template.json", // 7.8 experimental
-        };
-
-        /// <summary>
+		/// <summary>
         /// Map API default names for API's we are only supporting on the low level client first
         /// </summary>
         private static readonly Dictionary<string, string> LowLevelApiNameMapping = new Dictionary<string, string>
@@ -92,18 +62,12 @@ public static class CodeConfiguration
         public static readonly HashSet<string> EnableHighLevelCodeGen = new HashSet<string>();
 
         public static bool IsNewHighLevelApi(string apiFileName) =>
-            // if its explicitly ignored we know about it.
-            !IgnoredApis.Contains(apiFileName)
-            && !IgnoredApisHighLevel.Contains(apiFileName)
             // no requests with [MapsApi("filename.json")] found
-            && !HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""));
+            !HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""));
 
         public static bool IgnoreHighLevelApi(string apiFileName)
         {
-            //explicitly ignored
-            if (IgnoredApis.Contains(apiFileName) || IgnoredApisHighLevel.Contains(apiFileName)) return true;
-
-            //always generate already mapped requests
+			//always generate already mapped requests
 
             if (HighLevelApiNameMapping.ContainsKey(apiFileName.Replace(".json", ""))) return false;
 

src/ApiGenerator/Configuration/GeneratorLocations.cs

@@ -34,11 +34,11 @@ namespace ApiGenerator.Configuration
     public static class GeneratorLocations
     {
         // @formatter:off — disable formatter after this line
+		public static string OpenApiSpecFile { get; } = $@"{Root}OpenSearch.openapi.json";
+
         public static string OpenSearchNetFolder { get; } = $@"{Root}../../src/OpenSearch.Net/";
-        public static string LastDownloadedRef { get; } = Path.Combine(Root, "last_downloaded_version.txt");
 
         public static string OpenSearchClientFolder { get; } = $@"{Root}../../src/OpenSearch.Client/";
-        public static string RestSpecificationFolder { get; } = $@"{Root}RestSpecification/";
         // @formatter:on — enable formatter after this line
 
         public static string HighLevel(params string[] paths) => OpenSearchClientFolder + string.Join("/", paths);

src/ApiGenerator/Domain/ApiQueryParametersPatcher.cs

@@ -39,8 +39,7 @@ public static class ApiQueryParametersPatcher
         public static SortedDictionary<string, QueryParameters> Patch(
             string endpointName,
             IDictionary<string, QueryParameters> source,
-            IEndpointOverrides overrides,
-            bool checkCommon = true
+            IEndpointOverrides overrides
         )
         {
             if (source == null) return null;
@@ -54,32 +53,23 @@ public static SortedDictionary<string, QueryParameters> Patch(
             var obsoleteLookup = CreateObsoleteLookup(globalOverrides, overrides, declaredKeys);
 
             var patchedParams = new SortedDictionary<string, QueryParameters>();
-            var name = overrides?.GetType().Name ?? endpointName ?? "unknown";
-            foreach (var kv in source)
+            foreach (var (queryStringKey, value) in source)
             {
-                var queryStringKey = kv.Key;
-                kv.Value.QueryStringKey = queryStringKey;
+				value.QueryStringKey = queryStringKey;
 
-                if (checkCommon && RestApiSpec.CommonApiQueryParameters.Keys.Contains(queryStringKey))
-                {
-                    Generator.ApiGenerator.Warnings.Add($"key '{queryStringKey}' in {name} is already declared in _common.json");
-                    continue;
-                }
+                if (!renameLookup.TryGetValue(queryStringKey, out var preferredName)) preferredName = queryStringKey;
+                value.ClsName = CreateCSharpName(preferredName, endpointName);
 
-                if (!renameLookup.TryGetValue(queryStringKey, out var preferredName)) preferredName = kv.Key;
-                kv.Value.ClsName = CreateCSharpName(preferredName, endpointName);
+                if (skipList.Contains(queryStringKey)) value.Skip = true;
 
-                if (skipList.Contains(queryStringKey)) kv.Value.Skip = true;
+                if (partialList.Contains(queryStringKey)) value.RenderPartial = true;
 
-                if (partialList.Contains(queryStringKey)) kv.Value.RenderPartial = true;
-
-                if (obsoleteLookup.TryGetValue(queryStringKey, out var obsolete)) kv.Value.Obsolete = obsolete;
+                if (obsoleteLookup.TryGetValue(queryStringKey, out var obsolete)) value.Obsolete = obsolete;
 
                 //make sure source_enabled takes a boolean only
-                if (preferredName == "source_enabled") kv.Value.Type = "boolean";
-
+                if (preferredName == "source_enabled") value.Type = "boolean";
 
-                patchedParams[preferredName] = kv.Value;
+				patchedParams[preferredName] = value;
             }
 
             return patchedParams;

src/ApiGenerator/Domain/Code/CsharpNames.cs

@@ -31,7 +31,6 @@
 using System.Linq;
 using ApiGenerator.Configuration;
 using ApiGenerator.Generator;
-using CsQuery.ExtensionMethods.Internal;
 
 namespace ApiGenerator.Domain.Code
 {
@@ -73,7 +72,7 @@ string Replace(string original, string ns, string find, string replace, string[]
         public string RestSpecName { get; }
 
         /// <summary>
-        /// The pascal cased method name as loaded by <see cref="ApiEndpointFactory.FromFile"/>
+        /// The pascal cased method name as loaded by <see cref="ApiEndpointFactory.From"/>
         /// <pre>Uses <see cref="CodeConfiguration.ApiNameMapping"/> mapping of request implementations in the OSC code base</pre>
         /// </summary>
         public string MethodName { get; }

src/ApiGenerator/Domain/RestApiSpec.cs

@@ -43,19 +43,15 @@ public class EnumDescription
 
     public class RestApiSpec
     {
-        public string Commit { get; set; }
-
-        public static SortedDictionary<string, QueryParameters> CommonApiQueryParameters { get; set; }
-
-        public IDictionary<string, ApiEndpoint> Endpoints { get; set; }
+		public IDictionary<string, ApiEndpoint> Endpoints { get; set; }
 
         public ImmutableSortedDictionary<string, ReadOnlyCollection<ApiEndpoint>> EndpointsPerNamespaceLowLevel =>
             Endpoints.Values.GroupBy(e=>e.CsharpNames.Namespace)
                 .ToImmutableSortedDictionary(kv => kv.Key, kv => kv.ToList().AsReadOnly());
 
         public ImmutableSortedDictionary<string, ReadOnlyCollection<ApiEndpoint>> EndpointsPerNamespaceHighLevel =>
             Endpoints.Values
-                .Where(v => !CodeConfiguration.IgnoreHighLevelApi(v.FileName))
+                .Where(v => !CodeConfiguration.IgnoreHighLevelApi(v.Name))
                 .GroupBy(e => e.CsharpNames.Namespace)
                 .ToImmutableSortedDictionary(kv => kv.Key, kv => kv.ToList().AsReadOnly());
 
@@ -111,16 +107,7 @@ from part in e.Url.Parts
                     .DistinctBy(e => e.Name)
                     .ToList();
 
-                //TODO can be removed in 8.x
-                var versionType = _enumDescriptions.FirstOrDefault(f => f.Name == "VersionType");
-                if (versionType != null)
-                {
-                    var options = new List<string>(versionType.Options);
-                    options.Add("force");
-                    versionType.Options = options;
-                }
-
-                return _enumDescriptions;
+				return _enumDescriptions;
             }
         }
     }

src/ApiGenerator/Domain/Specification/ApiEndpoint.cs

@@ -40,10 +40,7 @@ namespace ApiGenerator.Domain.Specification
 {
     public class ApiEndpoint
     {
-        /// <summary> The filename of the spec describing the api endpoint </summary>
-        public string FileName { get; set; }
-
-        /// <summary> The original name as declared in the spec </summary>
+		/// <summary> The original name as declared in the spec </summary>
         public string Name { get; set; }
 
         /// <summary> The original namespace as declared in the spec </summary>

src/ApiGenerator/Domain/Specification/UrlInformation.cs

@@ -39,8 +39,8 @@ public class UrlInformation
     {
         public IDictionary<string, QueryParameters> Params { get; set; } = new SortedDictionary<string, QueryParameters>();
 
-        [JsonProperty("paths")]
-        private IReadOnlyCollection<string> OriginalPaths { get; set; }
+		[JsonProperty("paths")]
+		public IList<string> OriginalPaths { get; set; } = new List<string>();
 
         [JsonProperty("parts")]
         public IDictionary<string, UrlPart> OriginalParts { get; set; }

src/ApiGenerator/Extensions.cs

@@ -31,7 +31,6 @@
 using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
-using CsQuery.ExtensionMethods.Internal;
 
 namespace ApiGenerator
 {
@@ -64,10 +63,13 @@ public static string ToCamelCase(this string s)
             var pascal = s.ToPascalCase(true);
             if (pascal.Length <= 1) return pascal;
 
-            return pascal[0].ToLower() + pascal.Substring(1);
+            return char.ToLower(pascal[0]) + pascal.Substring(1);
         }
 
         public static string SplitPascalCase(this string s) =>
             Regex.Replace(s, "([A-Z]+[a-z]*)", " $1").Trim();
-    }
+
+		public static bool IsNullOrEmpty(this string s) =>
+			string.IsNullOrEmpty(s);
+	}
 }