Added Improvements to the Features and Messaging to the GC Infra re: ASP.NET Benchmarks

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks - Linux.csv → src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux.csv

@@ -1,9 +1,6 @@
 Legend,Base CommandLine
-FortunesEf_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --scenario fortunes_ef --application.options.collectCounters true --property os=windows --property arch=x64 --profile aspnet-citrine-win
-JsonMin_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --scenario mapaction --application.framework net8.0 --application.options.collectCounters true --property os=windows --property arch=x64 --profile aspnet-citrine-win
 FortunesEf_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --scenario fortunes_ef --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
 JsonMin_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --scenario mapaction --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
 FortunesEf_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --scenario fortunes_ef --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
 JsonMin_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --scenario mapaction --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
-Stage1Grpc_Windows, crank --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-win-app --profile intel-lin-load --application.options.collectCounters true
 Stage1Grpc_Linux, crank --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-load2-app --profile amd-lin2-load --profile amd-lin2-db --application.options.collectCounters true

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.csv

@@ -1,2 +1,3 @@
 Legend,Base CommandLine
-Stage1Grpc_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-load2-app --profile amd-lin2-load --profile amd-lin2-db --application.options.collectCounters true
+JsonMvc_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario mvc --profile intel-win-app --profile intel-load2-load  --application.collectDependencies true --application.options.collectCounters true 
+FortunesEf_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_ef --profile intel-win-app --profile intel-load-load --profile intel-db-db  --application.options.collectCounters true --application.collectDependencies true 

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.yaml

@@ -3,15 +3,18 @@ runs:
     corerun: C:\CoreRuns\Empty\
     environment_variables:
       COMPlus_GCServer: 1
+    framework_version: net6.0
   run:
     corerun: C:\CoreRuns\Empty\
     environment_variables:
       COMPlus_GCServer: 1
 environment:
   environment_variables: {}
   default_max_seconds: 300
+  framework_version: net8.0
 benchmark_settings:
   benchmark_file: C:\InfraRuns\RunNew_All\Suites\ASPNETBenchmarks\ASPNetBenchmarks.csv
+  additional_arguments: --chart --chart-type hex
 output:
   path: C:\InfraRuns\RunNew_All\ASPNetBenchmarks
   columns: 

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs

@@ -6,13 +6,13 @@ namespace GC.Infrastructure.Core.CommandBuilders
 {
     public static class ASPNetBenchmarksCommandBuilder 
     {
-        public static (string, string) Build(ASPNetBenchmarksConfiguration configuration, KeyValuePair<string, Run> run, KeyValuePair<string, string> baseConfiguration, OS os)
+        public static (string, string) Build(ASPNetBenchmarksConfiguration configuration, KeyValuePair<string, Run> run, KeyValuePair<string, string> benchmarkNameToCommand, OS os)
         {
             string processName = "crank";
             StringBuilder commandStringBuilder = new();
 
             // Load the base configuration.
-            commandStringBuilder.Append(baseConfiguration.Value);
+            commandStringBuilder.Append(benchmarkNameToCommand.Value);
 
             // Environment Variables.
             // Add the environment variables from the configuration.
@@ -33,7 +33,20 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration
 
             foreach (var env in environmentVariables)
             {
-                commandStringBuilder.Append($" --application.environmentVariables {env.Key}={env.Value} ");
+                string variable = env.Value;
+
+                // Check if the log file is specified, also add the fact that we want to retrieve the log file back.
+                // This log file should be named in concordance with the name of the run and the benchmark.
+                if (string.CompareOrdinal(env.Key, "DOTNET_GCLogFile") == 0 ||
+                    string.CompareOrdinal(env.Key, "COMPlus_GCLogFile") == 0)
+                {
+                    string fileNameOfLog = Path.GetFileName(env.Value);
+                    commandStringBuilder.Append( $" --application.options.downloadFiles \"*{fileNameOfLog}.log\" " );
+                    string fileName = Path.GetFileNameWithoutExtension(env.Value);
+                    commandStringBuilder.Append( $" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_GCLog")}\" " );
+                }
+
+                commandStringBuilder.Append($" --application.environmentVariables {env.Key}={variable} ");
             }
 
             // Trace Collection. 
@@ -69,19 +82,42 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration
                 }
 
                 // Add name of output.
-                commandStringBuilder.Append($" --application.options.traceOutput {Path.Combine(configuration.Output.Path, run.Key, (baseConfiguration.Key + "." + collectType)) + traceFileSuffix}");
+                commandStringBuilder.Append($" --application.options.traceOutput {Path.Combine(configuration.Output.Path, run.Key, (benchmarkNameToCommand.Key + "." + collectType)) + traceFileSuffix}");
+            }
+
+            // Add any additional arguments specified.
+            if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments))
+            {
+                commandStringBuilder.Append($" {configuration.benchmark_settings.additional_arguments} ");
             }
 
-            commandStringBuilder.Append($" --application.framework net8.0 ");
+            string frameworkVersion = configuration.Environment.framework_version;
+            // Override the framework version if it's specified at the level of the run.
+            if (!string.IsNullOrEmpty(run.Value.framework_version))
+            {
+                frameworkVersion = run.Value.framework_version;
+            }
+            commandStringBuilder.Append($" --application.framework {frameworkVersion} ");
 
-            string corerunToSend = run.Value.corerun.EndsWith("\\") ? run.Value.corerun.Remove(run.Value.corerun.Length - 1) : run.Value.corerun;
-            commandStringBuilder.Append($" --application.options.outputFiles {Path.Combine(Path.GetDirectoryName(corerunToSend), "*.*" )}");
+            string artifactsToUpload = run.Value.corerun!;
 
-            // Get the log.
+            // If the corerun specified is a directory, upload the entire directory.
+            // Else, we upload just the file.
+            if (Directory.Exists(run.Value.corerun!))
+            {
+                artifactsToUpload = Path.Combine(artifactsToUpload, "*.*");
+            }
+            commandStringBuilder.Append($" --application.options.outputFiles {artifactsToUpload} ");
+
+            // Get the logs.
             commandStringBuilder.Append(" --application.options.downloadOutput true ");
+            commandStringBuilder.Append($" --application.options.downloadOutputOutput {Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_{run.Key}.output.log")} ");
+
             commandStringBuilder.Append(" --application.options.downloadBuildLog true ");
+            commandStringBuilder.Append($" --application.options.downloadBuildLogOutput {Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_{run.Key}.build.log")} ");
+
 
-            commandStringBuilder.Append($" --json {Path.Combine(configuration.Output.Path, run.Key, $"{baseConfiguration.Key}_{run.Key}.json")}");
+            commandStringBuilder.Append($" --json {Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_{run.Key}.json")}");
             return (processName, commandStringBuilder.ToString());
         }
     }

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs

@@ -11,17 +11,20 @@ public sealed class ASPNetBenchmarksConfiguration : ConfigurationBase
     public sealed class Run : RunBase
     {
         public string? corerun { get; set; }
+        public string? framework_version { get; set; }
     }
 
     public class Environment
     {
         public Dictionary<string, string> environment_variables { get; set; } = new();
         public uint default_max_seconds { get; set; } = 300;
+        public string framework_version { get; set; } = "net8.0";
     }
 
     public class BenchmarkSettings
     {
         public string benchmark_file { get; set; }
+        public string additional_arguments { get; set; } = "";
     }
 
     public class Output : OutputBase { }

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksAnalyzeCommand.cs

@@ -21,6 +21,9 @@ public sealed class AspNetBenchmarkAnalyzeSettings : CommandSettings
 
         public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBenchmarkAnalyzeSettings settings)
         {
+            AnsiConsole.Write(new Rule("ASP.NET Benchmarks Analyzer"));
+            AnsiConsole.WriteLine();
+
             ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(AspNetBenchmarksCommand));
             ASPNetBenchmarksConfiguration configuration = ASPNetBenchmarksConfigurationParser.Parse(settings.ConfigurationPath);
             // Parse the CSV file for the information.
@@ -38,7 +41,17 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe
                 configurationToCommand[line[0]] = line[1];
             }
 
-            Dictionary<string, List<MetricResult>> results = ExecuteAnalysis(configuration, configurationToCommand, new());
+            Dictionary<string, List<MetricResult>> result = ExecuteAnalysis(configuration, configurationToCommand, new());
+            if (result.Count == 0)
+            {
+                AnsiConsole.MarkupLine($"[bold green] No report generated since there were no results to compare. [/]");
+            }
+
+            else
+            {
+                AnsiConsole.MarkupLine($"[bold green] Report generated at: {Path.Combine(configuration.Output.Path, "Results.md")} [/]");
+            }
+
             return 0;
         }
 
@@ -47,6 +60,13 @@ public static Dictionary<string, List<MetricResult>> ExecuteAnalysis(ASPNetBench
             // Benchmark to Run to Path. 
             Dictionary<string, List<string>> benchmarkToRunToPaths = new();
 
+            bool singleRun = configuration.Runs.Count == 1;
+            // Don't generate a report in case of a single report.
+            if (singleRun)
+            {
+                return new();
+            }
+
             // For each Run, grab the paths of each of the benchmarks.
             string outputPath = configuration.Output.Path;
             foreach (var c in configuration.Runs)
@@ -64,7 +84,6 @@ public static Dictionary<string, List<MetricResult>> ExecuteAnalysis(ASPNetBench
                 }
             }
 
-            // Launch new process.
             Dictionary<string, string> benchmarkToComparisons = new();
             Dictionary<string, List<MetricResult>> metricResults = new();
 
@@ -102,6 +121,7 @@ public static Dictionary<string, List<MetricResult>> ExecuteAnalysis(ASPNetBench
 
             using (StreamWriter sw = new StreamWriter(Path.Combine(configuration.Output.Path, "Results.md")))
             {
+                // Ignore the summary section in case there is only one run.
                 sw.WriteLine("# Summary");
 
                 var topLevelSummarySet = new HashSet<string>(new List<string> { "Working Set (MB)", "Private Memory (MB)", "Requests/sec", "Mean Latency (MSec)", "Latency 50th (MSec)", "Latency 75th (MSec)", "Latency 90th (MSec)", "Latency 99th (MSec)" });
@@ -142,7 +162,7 @@ public static Dictionary<string, List<MetricResult>> ExecuteAnalysis(ASPNetBench
 
                 foreach (var benchmark in benchmarkToComparisons)
                 {
-                    sw.WriteLine($"- [{benchmark.Key}](##{benchmark.Key})");
+                    sw.WriteLine($"- [{benchmark.Key}](#{benchmark.Key.ToLower().Replace(" ", "-")})");
                 }
 
                 sw.WriteLine();