Add ability to generate only a subset of operations (#278)

Signed-off-by: Thomas Farr <tsfarr@amazon.com>
opensearch-project · Jul 12, 2023 · d2077f4 · d2077f4
1 parent b3312fc
commit d2077f4
Show file tree

Hide file tree

Showing 34 changed files with 1,123 additions and 345 deletions.
diff --git a/src/ApiGenerator/ApiGenerator.csproj b/src/ApiGenerator/ApiGenerator.csproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -9,6 +9,7 @@
     <PreserveCompilationContext>true</PreserveCompilationContext>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Glob" Version="1.1.9" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.6.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="NSwag.Core.Yaml" Version="13.19.0" />

diff --git a/src/ApiGenerator/Configuration/CodeConfiguration.cs b/src/ApiGenerator/Configuration/CodeConfiguration.cs
@@ -31,11 +31,19 @@
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using GlobExpressions;
 
 namespace ApiGenerator.Configuration
 {
     public static class CodeConfiguration
-    {
+	{
+		private static readonly Glob[] OperationsToInclude =
+		{
+			// e.g. new Glob("nodes.*"),
+		};
+
+		public static bool IncludeOperation(string name) => OperationsToInclude.Any(g => g.IsMatch(name));
+
 		/// <summary>
         /// Map API default names for API's we are only supporting on the low level client first
         /// </summary>

diff --git a/src/ApiGenerator/Configuration/GeneratorLocations.cs b/src/ApiGenerator/Configuration/GeneratorLocations.cs
@@ -39,10 +39,15 @@ public static class GeneratorLocations
         public static string OpenSearchNetFolder { get; } = $@"{Root}../../src/OpenSearch.Net/";
 
         public static string OpenSearchClientFolder { get; } = $@"{Root}../../src/OpenSearch.Client/";
-        // @formatter:on — enable formatter after this line
 
-        public static string HighLevel(params string[] paths) => OpenSearchClientFolder + string.Join("/", paths);
-        public static string LowLevel(params string[] paths) => OpenSearchNetFolder + string.Join("/", paths);
+		public static string LowLevelGeneratedFolder { get; } = $"{OpenSearchNetFolder}_Generated/";
+
+		public static string HighLevelGeneratedFolder { get; } = $"{OpenSearchClientFolder}_Generated/";
+
+		// @formatter:on — enable formatter after this line
+
+        public static string HighLevel(params string[] paths) => HighLevelGeneratedFolder + string.Join("/", paths);
+        public static string LowLevel(params string[] paths) => LowLevelGeneratedFolder + string.Join("/", paths);
 
         public static readonly Assembly Assembly = typeof(Generator.ApiGenerator).Assembly;
 

diff --git a/src/ApiGenerator/Generator/ApiEndpointFactory.cs b/src/ApiGenerator/Generator/ApiEndpointFactory.cs
@@ -141,7 +141,7 @@ private static string GetOpenSearchType(JsonSchema schema)
 	        while (schema.HasReference) schema = schema.Reference;
 
 			if (schema.GetExtension("x-data-type") is string dataType)
-				return dataType;
+				return dataType == "array" ? "list" : dataType;
 
 	        return schema.Type switch
 	        {

diff --git a/src/ApiGenerator/Generator/ApiGenerator.cs b/src/ApiGenerator/Generator/ApiGenerator.cs
@@ -29,9 +29,10 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
- using System.Threading;
- using System.Threading.Tasks;
+using System.Threading;
+using System.Threading.Tasks;
 using ApiGenerator.Configuration;
 using ApiGenerator.Domain;
 using ApiGenerator.Generator.Razor;
@@ -42,48 +43,51 @@ namespace ApiGenerator.Generator
 {
     public class ApiGenerator
     {
-        public static List<string> Warnings { get; private set; } = new List<string>();
+        public static List<string> Warnings { get; private set; } = new();
 
         public static async Task Generate(bool lowLevelOnly, RestApiSpec spec, CancellationToken token)
         {
-            static async Task DoGenerate(ICollection<RazorGeneratorBase> generators, RestApiSpec restApiSpec, bool highLevel, CancellationToken token)
+			async Task DoGenerate(IReadOnlyCollection<RazorGeneratorBase> generators, bool highLevel)
             {
                 var pbarOpts = new ProgressBarOptions { ProgressCharacter = '─', BackgroundColor = ConsoleColor.Yellow };
                 var message = $"Generating {(highLevel ? "high" : "low")} level code";
                 using var pbar = new ProgressBar(generators.Count, message, pbarOpts);
                 foreach (var generator in generators)
                 {
                     pbar.Message = "Generating " + generator.Title;
-                    await generator.Generate(restApiSpec, pbar, token);
+                    await generator.Generate(spec, pbar, token);
                     pbar.Tick("Generated " + generator.Title);
                 }
             }
 
+			RecursiveDelete(GeneratorLocations.LowLevelGeneratedFolder);
+			await DoGenerate(
+				new RazorGeneratorBase[] {
+					//low level client
+					new LowLevelClientInterfaceGenerator(),
+					new LowLevelClientImplementationGenerator(),
+					new RequestParametersGenerator(),
+					new EnumsGenerator()
+				},
+				highLevel: false
+			);
 
-            var lowLevelGenerators = new List<RazorGeneratorBase>
-            {
-                //low level client
-                new LowLevelClientInterfaceGenerator(),
-                new LowLevelClientImplementationGenerator(),
-                new RequestParametersGenerator(),
-                new EnumsGenerator(),
-                new ApiUrlsLookupsGenerator(),
-            };
-
-            var highLevelGenerators = new List<RazorGeneratorBase>
-            {
-                //high level client
-                new HighLevelClientInterfaceGenerator(),
-                new HighLevelClientImplementationGenerator(),
-                new DescriptorsGenerator(),
-                new RequestsGenerator(),
-            };
+			if (lowLevelOnly) return;
 
-            await DoGenerate(lowLevelGenerators, spec, highLevel: false, token);
-            if (!lowLevelOnly)
-                await DoGenerate(highLevelGenerators, spec, highLevel: true, token);
-
-        }
+			RecursiveDelete(GeneratorLocations.HighLevelGeneratedFolder);
+			await DoGenerate(
+				new RazorGeneratorBase[]
+				{
+					//high level client
+					new ApiUrlsLookupsGenerator(),
+					new HighLevelClientInterfaceGenerator(),
+					new HighLevelClientImplementationGenerator(),
+					new DescriptorsGenerator(),
+					new RequestsGenerator(),
+				},
+				highLevel: true
+			);
+		}
 
 		public static async Task<RestApiSpec> CreateRestApiSpecModel(CancellationToken token = default)
 		{
@@ -93,10 +97,18 @@ public static async Task<RestApiSpec> CreateRestApiSpecModel(CancellationToken t
 				.Select(kv => new { HttpPath = kv.Key, PathItem = kv.Value })
 				.SelectMany(p => p.PathItem.Select(kv => new { p.HttpPath, p.PathItem, HttpMethod = kv.Key, Operation = kv.Value }))
 				.GroupBy(o => o.Operation.ExtensionData["x-operation-group"].ToString())
+				.Where(o => CodeConfiguration.IncludeOperation(o.Key))
 				.Select(o => ApiEndpointFactory.From(o.Key, o.Select(i => (i.HttpPath, i.PathItem, i.HttpMethod, i.Operation)).ToList()))
 				.ToImmutableSortedDictionary(e => e.Name, e => e);
 
 			return new RestApiSpec { Endpoints = endpoints };
 		}
+
+		private static void RecursiveDelete(string path)
+		{
+			if (!Directory.Exists(path)) return;
+
+			Directory.Delete(path, true);
+		}
     }
 }
diff --git a/src/ApiGenerator/Generator/Razor/ApiUrlsLookupsGenerator.cs b/src/ApiGenerator/Generator/Razor/ApiUrlsLookupsGenerator.cs
@@ -41,7 +41,7 @@ public class ApiUrlsLookupsGenerator : RazorGeneratorBase
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
             var view = ViewLocations.HighLevel("Requests", "ApiUrlsLookup.cshtml");
-            var target = GeneratorLocations.HighLevel("_Generated", "ApiUrlsLookup.generated.cs");
+            var target = GeneratorLocations.HighLevel("ApiUrlsLookup.cs");
 
             await DoRazor(spec, view, target, token);
         }

diff --git a/src/ApiGenerator/Generator/Razor/DescriptorsGenerator.cs b/src/ApiGenerator/Generator/Razor/DescriptorsGenerator.cs
@@ -42,11 +42,7 @@ public class DescriptorsGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            // Delete existing files
-            foreach (var file in Directory.GetFiles(GeneratorLocations.OpenSearchClientFolder, "Descriptors.*.cs"))
-                File.Delete(file);
-
-            var view = ViewLocations.HighLevel("Descriptors", "RequestDescriptorBase.cshtml");
+			var view = ViewLocations.HighLevel("Descriptors", "RequestDescriptorBase.cshtml");
             var target = GeneratorLocations.HighLevel("Descriptors.cs");
             await DoRazor(spec, view, target, token);
 

diff --git a/src/ApiGenerator/Generator/Razor/EnumsGenerator.cs b/src/ApiGenerator/Generator/Razor/EnumsGenerator.cs
@@ -40,8 +40,8 @@ public class EnumsGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            var view = ViewLocations.LowLevel("Enums.Generated.cshtml");
-            var target = GeneratorLocations.LowLevel("Api", "Enums.Generated.cs");
+            var view = ViewLocations.LowLevel("Enums.cshtml");
+            var target = GeneratorLocations.LowLevel("Api", "Enums.cs");
 
             await DoRazor(spec, view, target, token);
         }

diff --git a/src/ApiGenerator/Generator/Razor/HighLevelClientImplementationGenerator.cs b/src/ApiGenerator/Generator/Razor/HighLevelClientImplementationGenerator.cs
@@ -43,11 +43,7 @@ public class HighLevelClientImplementationGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            // Delete existing files
-            foreach (var file in Directory.GetFiles(GeneratorLocations.OpenSearchClientFolder, "OpenSearchClient.*.cs"))
-                File.Delete(file);
-
-            var view = ViewLocations.HighLevel("Client", "Implementation", "OpenSearchClient.cshtml");
+			var view = ViewLocations.HighLevel("Client", "Implementation", "OpenSearchClient.cshtml");
             var target = GeneratorLocations.HighLevel($"OpenSearchClient.{CsharpNames.RootNamespace}.cs");
             await DoRazor(spec, view, target, token);
 

diff --git a/src/ApiGenerator/Generator/Razor/HighLevelClientInterfaceGenerator.cs b/src/ApiGenerator/Generator/Razor/HighLevelClientInterfaceGenerator.cs
@@ -41,7 +41,7 @@ public class HighLevelClientInterfaceGenerator : RazorGeneratorBase
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
             var view = ViewLocations.HighLevel("Client", "Interface", "IOpenSearchClient.cshtml");
-            var target = GeneratorLocations.HighLevel("IOpenSearchClient.Generated.cs");
+            var target = GeneratorLocations.HighLevel("IOpenSearchClient.cs");
 
             await DoRazor(spec, view, target, token);
         }

diff --git a/src/ApiGenerator/Generator/Razor/LowLevelClientImplementationGenerator.cs b/src/ApiGenerator/Generator/Razor/LowLevelClientImplementationGenerator.cs
@@ -43,11 +43,7 @@ public class LowLevelClientImplementationGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            // Delete existing files
-            foreach (var file in Directory.GetFiles(GeneratorLocations.OpenSearchNetFolder, "OpenSearchLowLevelClient.*.cs"))
-                File.Delete(file);
-
-            var view = ViewLocations.LowLevel("Client", "Implementation", "OpenSearchLowLevelClient.cshtml");
+			var view = ViewLocations.LowLevel("Client", "Implementation", "OpenSearchLowLevelClient.cshtml");
             var target = GeneratorLocations.LowLevel($"OpenSearchLowLevelClient.{CsharpNames.RootNamespace}.cs");
             await DoRazor(spec, view, target, token);
 

diff --git a/src/ApiGenerator/Generator/Razor/LowLevelClientInterfaceGenerator.cs b/src/ApiGenerator/Generator/Razor/LowLevelClientInterfaceGenerator.cs
@@ -41,7 +41,7 @@ public class LowLevelClientInterfaceGenerator : RazorGeneratorBase
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
             var view = ViewLocations.LowLevel("Client", "Interface", "IOpenSearchLowLevelClient.cshtml");
-            var target = GeneratorLocations.LowLevel("IOpenSearchLowLevelClient.Generated.cs");
+            var target = GeneratorLocations.LowLevel("IOpenSearchLowLevelClient.cs");
 
             await DoRazor(spec, view, target, token);
         }

diff --git a/src/ApiGenerator/Generator/Razor/RazorGeneratorBase.cs b/src/ApiGenerator/Generator/Razor/RazorGeneratorBase.cs
@@ -93,7 +93,10 @@ protected static void WriteFormattedCsharpFile(string path, string contents)
             var tree = CSharpSyntaxTree.ParseText(contents);
             var root = tree.GetRoot().NormalizeWhitespace(indentation:"\t", "\n");
             contents = root.ToFullString();
-            File.WriteAllText(path, contents);
+
+			if (Directory.GetParent(path) is { Exists: false } dir) dir.Create();
+
+			File.WriteAllText(path, contents);
         }
 
         public abstract string Title { get; }

diff --git a/src/ApiGenerator/Generator/Razor/RequestParametersGenerator.cs b/src/ApiGenerator/Generator/Razor/RequestParametersGenerator.cs
@@ -42,11 +42,7 @@ public class RequestParametersGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            // Delete existing files
-            foreach (var file in Directory.GetFiles(GeneratorLocations.OpenSearchNetFolder, "RequestParameters.*.cs"))
-                File.Delete(file);
-
-            var view = ViewLocations.LowLevel("RequestParameters", "RequestParameters.cshtml");
+			var view = ViewLocations.LowLevel("RequestParameters", "RequestParameters.cshtml");
             string Target(string id) => GeneratorLocations.LowLevel("Api", "RequestParameters", $"RequestParameters.{id}.cs");
 
             var namespaced = spec.EndpointsPerNamespaceLowLevel.ToList();

diff --git a/src/ApiGenerator/Generator/Razor/RequestsGenerator.cs b/src/ApiGenerator/Generator/Razor/RequestsGenerator.cs
@@ -42,10 +42,6 @@ public class RequestsGenerator : RazorGeneratorBase
 
         public override async Task Generate(RestApiSpec spec, ProgressBar progressBar, CancellationToken token)
         {
-            // Delete existing files
-            foreach (var file in Directory.GetFiles(GeneratorLocations.OpenSearchClientFolder, "Requests.*.cs"))
-                File.Delete(file);
-
             var view = ViewLocations.HighLevel("Requests", "PlainRequestBase.cshtml");
             var target = GeneratorLocations.HighLevel("Requests.cs");
             await DoRazor(spec, view, target, token);