From 3c4653d41eadcf349ece96610b67e564bc3d13ea Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Tue, 14 Jul 2020 17:15:27 -0500 Subject: [PATCH] Create tests that verifies the Dockerfiles and templates are in sync --- build-and-test.ps1 | 4 ++ eng/common/templates/jobs/build-images.yml | 4 +- .../jobs/test-images-linux-client.yml | 13 ++--- .../stages/build-test-publish-repo.yml | 8 +++ .../steps/test-images-linux-client.yml | 29 ++++++---- .../Get-GeneratedDockerfiles.ps1 | 14 ++--- .../ScriptRunnerUpdater.cs | 13 +++-- tests/Microsoft.DotNet.Docker.Tests/Config.cs | 22 ++++---- .../DockerHelper.cs | 40 +------------- .../ExecuteHelper.cs | 54 +++++++++++++++++++ .../Microsoft.DotNet.Docker.Tests.csproj | 7 --- .../SampleImageTests.cs | 6 ++- .../TemplateTests.cs | 49 +++++++++++++++++ tests/run-tests.ps1 | 3 +- 14 files changed, 182 insertions(+), 84 deletions(-) create mode 100644 tests/Microsoft.DotNet.Docker.Tests/ExecuteHelper.cs create mode 100644 tests/Microsoft.DotNet.Docker.Tests/TemplateTests.cs diff --git a/build-and-test.ps1 b/build-and-test.ps1 index 9b4cef11ca..1fab93d5d8 100644 --- a/build-and-test.ps1 +++ b/build-and-test.ps1 @@ -24,6 +24,10 @@ param( [string[]]$TestCategories = @("runtime", "runtime-deps", "aspnet", "sdk", "sample", "image-size") ) +if ($Mode -eq "BuildAndTest" -or $Mode -eq "Test") { + & ./tests/run-tests.ps1 -TestCategories "pre-build" +} + if ($Mode -eq "BuildAndTest" -or $Mode -eq "Build") { # Build the product images & ./eng/common/build.ps1 ` diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index c6fdc04a72..130b544b35 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -9,7 +9,9 @@ parameters: jobs: - job: ${{ parameters.name }} condition: and(succeeded(), ${{ parameters.matrix }}) - dependsOn: GenerateBuildMatrix + dependsOn: + - PreBuildValidation + - GenerateBuildMatrix pool: ${{ parameters.pool }} strategy: matrix: $[ ${{ parameters.matrix }} ] diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml index fee60b95a1..85a3df2c13 100644 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ b/eng/common/templates/jobs/test-images-linux-client.yml @@ -3,16 +3,17 @@ parameters: pool: {} matrix: {} testJobTimeout: 60 + preBuildValidation: false jobs: - job: ${{ parameters.name }} - condition: and(succeeded(), ${{ parameters.matrix }}) - dependsOn: GenerateTestMatrix + ${{ if eq(parameters.preBuildValidation, 'false') }}: + condition: and(succeeded(), ${{ parameters.matrix }}) + dependsOn: GenerateTestMatrix + strategy: + matrix: $[ ${{ parameters.matrix }} ] pool: ${{ parameters.pool }} - strategy: - matrix: $[ ${{ parameters.matrix }} ] timeoutInMinutes: ${{ parameters.testJobTimeout }} steps: - template: ../steps/test-images-linux-client.yml parameters: - setupImageBuilder: false - setupTestRunner: true + preBuildValidation: ${{ parameters.preBuildValidation }} diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index b4f6621e64..d65bdd2fa3 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -23,6 +23,14 @@ stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) jobs: + - template: ../jobs/test-images-linux-client.yml + parameters: + name: PreBuildValidation + pool: + vmImage: $(defaultLinuxAmd64PoolImage) + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + - template: ../jobs/generate-matrix.yml parameters: matrixType: ${{ parameters.buildMatrixType }} diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 5ab1610f74..29332ca14f 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -1,3 +1,6 @@ +parameters: + preBuildValidation: false + steps: - template: init-docker-linux.yml parameters: @@ -8,11 +11,15 @@ steps: echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" optionalTestArgs="" - if [ "${{ eq(variables['System.TeamProject'], 'public') }}" == "False" ]; then - optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info/image-info.json" - fi - if [ "$REPOTESTARGS" != "" ]; then - optionalTestArgs="$optionalTestArgs $REPOTESTARGS" + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + optionalTestArgs="$optionalTestArgs -TestCategories pre-build" + else + if [ "${{ eq(variables['System.TeamProject'], 'public') }}" == "False" ]; then + optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info/image-info.json" + fi + if [ "$REPOTESTARGS" != "" ]; then + optionalTestArgs="$optionalTestArgs $REPOTESTARGS" + fi fi echo "##vso[task.setvariable variable=optionalTestArgs]$optionalTestArgs" displayName: Set Test Variables @@ -30,9 +37,10 @@ steps: -File $(engCommonRelativePath)/Invoke-WithRetry.ps1 "docker login -u $(acr.userName) -p $(BotAccount-dotnet-docker-acr-bot-password) $(acr.server)" displayName: Docker login - - template: ../steps/download-build-artifact.yml - parameters: - targetPath: $(Build.ArtifactStagingDirectory) + - ${{ if eq(parameters.preBuildValidation, 'false') }}: + - template: ../steps/download-build-artifact.yml + parameters: + targetPath: $(Build.ArtifactStagingDirectory) - script: > docker exec $(testRunner.container) pwsh -File $(testScriptPath) @@ -63,7 +71,10 @@ steps: searchFolder: $(Common.TestResultsDirectory) mergeTestResults: true publishRunAttachments: true - testRunTitle: $(dotnetVersion) $(osVariant) $(architecture) + ${{ if eq(parameters.preBuildValidation, 'false') }}: + testRunTitle: $(dotnetVersion) $(osVariant) $(architecture) + ${{ if eq(parameters.preBuildValidation, 'true') }}: + testRunTitle: Pre-Build Validation - script: docker rm -f $(testRunner.container) displayName: Cleanup TestRunner Container condition: always() diff --git a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 index 348e761105..cfd8577184 100644 --- a/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 +++ b/eng/dockerfile-templates/Get-GeneratedDockerfiles.ps1 @@ -1,20 +1,22 @@ #!/usr/bin/env pwsh param( - # Paths to the Dockerfiles to generate. - [string[]]$Paths + [switch]$Validate ) -if ($Paths) { - $pathArgs = " --path " + ($Paths -join " --path ") +if ($Validate) { + $customImageBuilderArgs = " --validate" } $repoRoot = (Get-Item "$PSScriptRoot").Parent.Parent.FullName $onDockerfilesGenerated = { param($ContainerName) - Exec "docker cp ${ContainerName}:/repo/src $repoRoot" + + if (-Not $Validate) { + Exec "docker cp ${ContainerName}:/repo/src $repoRoot" + } } & $PSScriptRoot/../common/Invoke-ImageBuilder.ps1 ` - -ImageBuilderArgs "generateDockerfiles --architecture * --os-type *$pathArgs" ` + -ImageBuilderArgs "generateDockerfiles --architecture '*' --os-type '*'$customImageBuilderArgs" ` -OnCommandExecuted $onDockerfilesGenerated diff --git a/eng/update-dependencies/ScriptRunnerUpdater.cs b/eng/update-dependencies/ScriptRunnerUpdater.cs index d88f9183f4..85557cc0aa 100644 --- a/eng/update-dependencies/ScriptRunnerUpdater.cs +++ b/eng/update-dependencies/ScriptRunnerUpdater.cs @@ -1,12 +1,13 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.DotNet.VersionTools.Dependencies; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; +using Microsoft.DotNet.VersionTools.Dependencies; namespace Dotnet.Docker { @@ -53,16 +54,22 @@ private void ExecuteScript() Trace.TraceInformation($"Executing '{_scriptPath}'"); // Support both execution within Windows 10, Nano Server and Linux environments. + Process process; try { - Process process = Process.Start("pwsh", _scriptPath); + process = Process.Start("pwsh", _scriptPath); process.WaitForExit(); } catch (Win32Exception) { - Process process = Process.Start("powershell", _scriptPath); + process = Process.Start("powershell", _scriptPath); process.WaitForExit(); } + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"Unable to successfully execute '{_scriptPath}'"); + } } } } diff --git a/tests/Microsoft.DotNet.Docker.Tests/Config.cs b/tests/Microsoft.DotNet.Docker.Tests/Config.cs index a79f1e3368..ba96118675 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/Config.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/Config.cs @@ -10,6 +10,9 @@ namespace Microsoft.DotNet.Docker.Tests { public static class Config { + private static Lazy Manifest { get; } = new Lazy(() => LoadManifest()); + + public static string SourceRepoRoot { get; } = Environment.GetEnvironmentVariable("SOURCE_REPO_ROOT") ?? string.Empty; public static bool IsHttpVerificationDisabled { get; } = Environment.GetEnvironmentVariable("DISABLE_HTTP_VERIFICATION") != null; public static bool PullImages { get; } = Environment.GetEnvironmentVariable("PULL_IMAGES") != null; @@ -17,21 +20,20 @@ public static class Config public static bool IsRunningInContainer { get; } = Environment.GetEnvironmentVariable("RUNNING_TESTS_IN_CONTAINER") != null; public static string RepoPrefix { get; } = Environment.GetEnvironmentVariable("REPO_PREFIX") ?? string.Empty; - public static string Registry { get; } = Environment.GetEnvironmentVariable("REGISTRY") ?? GetManifestRegistry(); + public static string Registry { get; } = + Environment.GetEnvironmentVariable("REGISTRY") ?? (string)Manifest.Value["registry"]; - private static string GetManifestRegistry() + private static bool GetIsNightlyRepo() { - string manifestJson = File.ReadAllText("manifest.json"); - JObject manifest = JObject.Parse(manifestJson); - return (string)manifest["registry"]; + string repo = (string)Manifest.Value["repos"][0]["name"]; + return repo.Contains("-nightly"); } - private static bool GetIsNightlyRepo() + private static JObject LoadManifest() { - string manifestJson = File.ReadAllText("manifest.json"); - JObject manifest = JObject.Parse(manifestJson); - string repo = (string)manifest["repos"][0]["name"]; - return repo.Contains("-nightly"); + string manifestPath = Path.Combine(SourceRepoRoot, "manifest.json"); + string manifestJson = File.ReadAllText(manifestPath); + return JObject.Parse(manifestJson); } } } diff --git a/tests/Microsoft.DotNet.Docker.Tests/DockerHelper.cs b/tests/Microsoft.DotNet.Docker.Tests/DockerHelper.cs index dc3382d181..3b9b582947 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/DockerHelper.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/DockerHelper.cs @@ -99,45 +99,7 @@ private static string Execute( } private static (Process Process, string StdOut, string StdErr) ExecuteProcess( - string args, ITestOutputHelper outputHelper) - { - Process process = new Process - { - EnableRaisingEvents = true, - StartInfo = - { - FileName = "docker", - Arguments = args, - RedirectStandardOutput = true, - RedirectStandardError = true, - } - }; - - StringBuilder stdOutput = new StringBuilder(); - process.OutputDataReceived += new DataReceivedEventHandler((sender, e) => stdOutput.AppendLine(e.Data)); - - StringBuilder stdError = new StringBuilder(); - process.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => stdError.AppendLine(e.Data)); - - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - process.WaitForExit(); - - string output = stdOutput.ToString().Trim(); - if (outputHelper != null && !string.IsNullOrWhiteSpace(output)) - { - outputHelper.WriteLine(output); - } - - string error = stdError.ToString().Trim(); - if (outputHelper != null && !string.IsNullOrWhiteSpace(error)) - { - outputHelper.WriteLine(error); - } - - return (process, output, error); - } + string args, ITestOutputHelper outputHelper) => ExecuteHelper.ExecuteProcess("docker", args, outputHelper); private string ExecuteWithLogging(string args, bool ignoreErrors = false, bool autoRetry = false) { diff --git a/tests/Microsoft.DotNet.Docker.Tests/ExecuteHelper.cs b/tests/Microsoft.DotNet.Docker.Tests/ExecuteHelper.cs new file mode 100644 index 0000000000..21ec732f3a --- /dev/null +++ b/tests/Microsoft.DotNet.Docker.Tests/ExecuteHelper.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Text; +using Xunit.Abstractions; + +namespace Microsoft.DotNet.Docker.Tests +{ + public static class ExecuteHelper + { + public static (Process Process, string StdOut, string StdErr) ExecuteProcess( + string fileName, string args, ITestOutputHelper outputHelper) + { + Process process = new Process + { + EnableRaisingEvents = true, + StartInfo = + { + FileName = fileName, + Arguments = args, + RedirectStandardOutput = true, + RedirectStandardError = true, + } + }; + + StringBuilder stdOutput = new StringBuilder(); + process.OutputDataReceived += new DataReceivedEventHandler((sender, e) => stdOutput.AppendLine(e.Data)); + + StringBuilder stdError = new StringBuilder(); + process.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => stdError.AppendLine(e.Data)); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + process.WaitForExit(); + + string output = stdOutput.ToString().Trim(); + if (outputHelper != null && !string.IsNullOrWhiteSpace(output)) + { + outputHelper.WriteLine(output); + } + + string error = stdError.ToString().Trim(); + if (outputHelper != null && !string.IsNullOrWhiteSpace(error)) + { + outputHelper.WriteLine(error); + } + + return (process, output, error); + } + } +} diff --git a/tests/Microsoft.DotNet.Docker.Tests/Microsoft.DotNet.Docker.Tests.csproj b/tests/Microsoft.DotNet.Docker.Tests/Microsoft.DotNet.Docker.Tests.csproj index 71cfd66859..c3ce541b9b 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/Microsoft.DotNet.Docker.Tests.csproj +++ b/tests/Microsoft.DotNet.Docker.Tests/Microsoft.DotNet.Docker.Tests.csproj @@ -18,13 +18,6 @@ PreserveNewest - - PreserveNewest - - - PreserveNewest - samples/%(RecursiveDir)%(Filename)%(Extension) - diff --git a/tests/Microsoft.DotNet.Docker.Tests/SampleImageTests.cs b/tests/Microsoft.DotNet.Docker.Tests/SampleImageTests.cs index d77a4623bc..9bed64c5ca 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/SampleImageTests.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/SampleImageTests.cs @@ -16,6 +16,8 @@ namespace Microsoft.DotNet.Docker.Tests [Trait("Category", "sample")] public class SampleImageTests { + private static readonly string s_samplesPath = Path.Combine(Config.SourceRepoRoot, "samples"); + public SampleImageTests(ITestOutputHelper outputHelper) { OutputHelper = outputHelper; @@ -80,7 +82,7 @@ public void VerifyComplexAppSample() { string appTag = SampleImageData.GetImageName("complexapp-local-app"); string testTag = SampleImageData.GetImageName("complexapp-local-test"); - string sampleFolder = $"samples/complexapp"; + string sampleFolder = Path.Combine(s_samplesPath, "complexapp"); string dockerfilePath = $"{sampleFolder}/Dockerfile"; string testContainerName = ImageData.GenerateContainerName("sample-complex-test"); string tempDir = null; @@ -141,7 +143,7 @@ private async Task VerifySampleAsync( { if (!imageData.IsPublished) { - string sampleFolder = $"samples/{imageType}"; + string sampleFolder = Path.Combine(s_samplesPath, imageType); string dockerfilePath = $"{sampleFolder}/Dockerfile.{imageData.DockerfileSuffix}"; DockerHelper.Build(image, dockerfilePath, contextDir: sampleFolder, pull: Config.PullImages); diff --git a/tests/Microsoft.DotNet.Docker.Tests/TemplateTests.cs b/tests/Microsoft.DotNet.Docker.Tests/TemplateTests.cs new file mode 100644 index 0000000000..d067d81e88 --- /dev/null +++ b/tests/Microsoft.DotNet.Docker.Tests/TemplateTests.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.DotNet.Docker.Tests +{ + [Trait("Category", "pre-build")] + public class TemplateTests + { + private ITestOutputHelper OutputHelper { get; } + + public TemplateTests(ITestOutputHelper outputHelper) + { + OutputHelper = outputHelper; + } + + [Fact] + public void VerifyDockerfileTemplates() + { + string generateDockerfilesScript = Path.Combine(Config.SourceRepoRoot, "eng", "dockerfile-templates", "Get-GeneratedDockerfiles.ps1"); + string powershellArgs = $"-File {generateDockerfilesScript} -Validate"; + (Process Process, string StdOut, string StdErr) executeResult; + + // Support both execution within Windows 10, Nano Server and Linux environments. + try + { + executeResult = ExecuteHelper.ExecuteProcess("pwsh", powershellArgs, OutputHelper); + } + catch (Win32Exception) + { + executeResult = ExecuteHelper.ExecuteProcess("powershell", powershellArgs, OutputHelper); + } + + if (executeResult.Process.ExitCode != 0) + { + OutputHelper.WriteLine( + $"The Dockerfiles are out of sync with the templates. Update the Dockerfiles by running `{generateDockerfilesScript}`."); + } + + Assert.Equal(0, executeResult.Process.ExitCode); + } + } +} diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index e5c251363f..06432ee1e4 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -14,7 +14,7 @@ param( [switch]$DisableHttpVerification, [switch]$PullImages, [string]$ImageInfoPath, - [ValidateSet("runtime", "runtime-deps", "aspnet", "sdk", "sample", "image-size")] + [ValidateSet("runtime", "runtime-deps", "aspnet", "sdk", "pre-build", "sample", "image-size")] [string[]]$TestCategories = @("runtime", "runtime-deps", "aspnet", "sdk") ) @@ -95,6 +95,7 @@ Try { $env:REGISTRY = $Registry $env:REPO_PREFIX = $RepoPrefix $env:IMAGE_INFO_PATH = $ImageInfoPath + $env:SOURCE_REPO_ROOT = (Get-Item "$PSScriptRoot").Parent.FullName $env:DOTNET_CLI_TELEMETRY_OPTOUT = 1 $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1