Small build improvements (#1646)

* Allow running an arbitrary dotnet sample app from Nuke

i.e. it doesn't have to be part of the solution

* Automatically build the "multiPackageProject" list from PackageVersionsGeneratorDefinitions.json

There's no point building these twice, so previously we had a list of sample projects to not build. This simplifies things somewhat by loading the list dynamically

* Try using a unique docker network per build to avoid issues with concurrent builds on arm64

* Upload artifacts to azure (for use in AAS extension automation)

The latest file list can be downloaded from:

https://apmdotnetci.blob.core.windows.net/apm-dotnet-ci-artifiacts-master/index.txt

and the latest commit sha of the uploads can be downloaded from

https://apmdotnetci.blob.core.windows.net/apm-dotnet-ci-artifiacts-master/sha.txt

The blobs are stored in a subdirectory

* Only push assets to Azure from master

* Pull Azure storage location from variable

.azure-pipelines/ultimate-pipeline.yml

@@ -576,6 +576,7 @@ stages:
         TestAllPackageVersions: true
         publishTargetFramework: net5.0
         baseImage: debian
+        COMPOSE_PROJECT_NAME: ddtrace_$(Build.BuildNumber)
 
       pool:
         name: Arm64
@@ -873,18 +874,13 @@ stages:
           )
         )
 
-
-
-
-- stage: upload_to_feed
+- stage: upload_to_azure
   condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'))
   dependsOn: [package_windows, build_linux, build_arm64, dotnet_tool]
   jobs:
     - job: upload
-
       pool:
         vmImage: ubuntu-18.04
-
       steps:
 
         - task: DownloadPipelineArtifact@2
@@ -936,7 +932,47 @@ stages:
           displayName: Publish release artifacts
           artifact: $(tracer_version)-release-artifacts
 
-        # We don't include the MSI artifacts as they're not signed
+        # We don't include the MSIs in the artifact upload as they're
+        # not signed, but we do include them in the push to Azure
+        - task: DownloadPipelineArtifact@2
+          displayName: Download x64 MSI
+          inputs:
+            artifact: windows-msi-x64
+            path: $(Build.ArtifactStagingDirectory)
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download x86 MSI
+          inputs:
+            artifact: windows-msi-x86
+            path: $(Build.ArtifactStagingDirectory)
+
+        - bash: |
+            az storage blob upload-batch \
+              --destination "$(AZURE_STORAGE_CONTAINER_NAME)" \
+              --destination-path "$(Build.SourceVersion)" \
+              --source "$(Build.ArtifactStagingDirectory)"
+          displayName: Upload blobs to Azure
+          condition: and(succeeded(), eq(variables.isMainBranch, true), ne(variables['isNgenTestBuild'], 'True'))
+          env:
+            AZURE_STORAGE_ACCOUNT: $(AZURE_STORAGE_ACCOUNT_NAME)
+            AZURE_STORAGE_SAS_TOKEN: $(AZURE_STORAGE_SHARED_ACCESS_TOKEN)
+
+        - bash: ls "$(Build.ArtifactStagingDirectory)" > index.txt
+          displayName: Write file list to index.txt
+
+        - bash: echo "$(Build.SourceVersion)" > sha.txt
+          displayName: Write commit hash to sha.txt
+
+        - bash: |
+            az storage blob upload --container-name "$(AZURE_STORAGE_CONTAINER_NAME)" --file "index.txt" --name "index.txt"
+            az storage blob upload --container-name "$(AZURE_STORAGE_CONTAINER_NAME)" --file "sha.txt" --name "sha.txt"
+          displayName: Upload indexes to Azure
+          condition: and(succeeded(), eq(variables.isMainBranch, true), ne(variables['isNgenTestBuild'], 'True'))
+          env:
+            AZURE_STORAGE_ACCOUNT: $(AZURE_STORAGE_ACCOUNT_NAME)
+            AZURE_STORAGE_SAS_TOKEN: $(AZURE_STORAGE_SHARED_ACCESS_TOKEN)
+
+
 
 - stage: throughput
   condition: and(succeeded(), ne(variables['isNgenTestBuild'], 'True'))

build/_build/Build.Steps.cs

@@ -4,7 +4,7 @@
 using System.Linq;
 using System.Net;
 using System.Runtime.InteropServices;
-using System.Text.RegularExpressions;
+using System.Text.Json;
 using Nuke.Common;
 using Nuke.Common.IO;
 using Nuke.Common.ProjectModel;
@@ -968,33 +968,29 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                 "NLog10LogsInjection.NullReferenceException",
                 "Sandbox.ManualTracing",
                 "StackExchange.Redis.AssemblyConflict.LegacyProject",
-                "MismatchedTracerVersions"
+                "Samples.OracleMDA", // We don't test these yet
+                "Samples.OracleMDA.Core", // We don't test these yet
+                "MismatchedTracerVersions",
             };
 
             // These sample projects are built using RestoreAndBuildSamplesForPackageVersions
             // so no point building them now
-            // TODO: Load this list dynamically
-            var multiApiProjects = new[]
+            List<string> multiPackageProjects;
+            var samplesFile = RootDirectory / "build" / "PackageVersionsGeneratorDefinitions.json";
+            using (var fs = File.OpenRead(samplesFile))
             {
-                "Samples.CosmosDb",
-                "Samples.MongoDB",
-                "Samples.Elasticsearch",
-                "Samples.Elasticsearch.V5",
-                "Samples.Kafka",
-                "Samples.Npgsql",
-                "Samples.RabbitMQ",
-                "Samples.SqlServer",
-                "Samples.Microsoft.Data.SqlClient",
-                "Samples.StackExchange.Redis",
-                "Samples.ServiceStack.Redis",
-                // "Samples.MySql", - the "non package version" is _ALSO_ tested separately
-                "Samples.Microsoft.Data.Sqlite",
-                "Samples.OracleMDA",
-                "Samples.OracleMDA.Core",
-                "Samples.XUnitTests",
-                "Samples.NUnitTests",
-                "Samples.MSTestTests",
-            };
+                var json = JsonDocument.Parse(fs);
+                multiPackageProjects = json.RootElement
+                                       .EnumerateArray()
+                                       .Select(e => e.GetProperty("SampleProjectName").GetString())
+                                       .Distinct()
+                                       .Where(name => name switch
+                                        {
+                                            "Samples.MySql" => false, // the "non package version" is _ALSO_ tested separately
+                                            _ => true
+                                        })
+                                       .ToList();
+            }
 
             var projectsToBuild = sampleProjects
                 .Concat(securitySampleProjects)
@@ -1011,7 +1007,7 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                         "Samples.AspNetCore2" => Framework == TargetFramework.NETCOREAPP2_1,
                         "Samples.AspNetCore5" => Framework == TargetFramework.NET5_0 || Framework == TargetFramework.NETCOREAPP3_1 || Framework == TargetFramework.NETCOREAPP3_0,
                         var name when projectsToSkip.Contains(name) => false,
-                        var name when multiApiProjects.Contains(name) => false,
+                        var name when multiPackageProjects.Contains(name) => false,
                         _ when !string.IsNullOrWhiteSpace(SampleName) => project?.Name?.Contains(SampleName) ?? false,
                         _ => true,
                     };

build/_build/Build.Utilities.cs

@@ -145,12 +145,20 @@ partial class Build
                 }
             }
 
-            Logger.Info($"Running sample '{SampleName}'");
-
-            var project = Solution.GetProject(SampleName)?.Path;
-            if (project is null)
+            string project = Solution.GetProject(SampleName)?.Path;
+            if (project is not null)
+            {
+                Logger.Info($"Running sample '{SampleName}'");
+            }
+            else if(System.IO.File.Exists(SampleName))
+            {
+                project = SampleName;
+                Logger.Info($"Running project '{SampleName}'");
+            }
+            else
             {
-                throw new Exception($"Could not find project in solution with name '{SampleName}'");
+                throw new Exception($"Could not find project in solution with name '{SampleName}', " +
+                                                    "and no project file with the provided path exists");
             }
 
             DotNetBuild(s => s