diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesCommand.cs b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesCommand.cs new file mode 100644 index 00000000..8a706bc4 --- /dev/null +++ b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesCommand.cs @@ -0,0 +1,154 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Cottle; +using Cottle.Exceptions; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ViewModel; + +namespace Microsoft.DotNet.ImageBuilder.Commands +{ + [Export(typeof(ICommand))] + public class GenerateDockerfilesCommand : ManifestCommand + { + private readonly DocumentConfiguration _config = new DocumentConfiguration + { + BlockBegin = "{{", + BlockContinue = "^", + BlockEnd = "}}", + Escape = '@', + Trimmer = DocumentConfiguration.TrimNothing + }; + + private readonly IEnvironmentService _environmentService; + + [ImportingConstructor] + public GenerateDockerfilesCommand(IEnvironmentService environmentService) : base() + { + _environmentService = environmentService ?? throw new ArgumentNullException(nameof(environmentService)); + } + + public override async Task ExecuteAsync() + { + Logger.WriteHeading("GENERATING DOCKERFILES"); + + List outOfSyncDockerfiles = new List(); + List invalidTemplates = new List(); + + IEnumerable platforms = Manifest.GetFilteredPlatforms() + .Where(platform => platform.DockerfileTemplate != null); + foreach (PlatformInfo platform in platforms) + { + Logger.WriteSubheading($"Generating '{platform.DockerfilePath}' from '{platform.DockerfileTemplate}'"); + + string template = await File.ReadAllTextAsync(platform.DockerfileTemplate); + if (Options.IsVerbose) + { + Logger.WriteMessage($"Template:{Environment.NewLine}{template}"); + } + + await GenerateDockerfileAsync(template, platform, outOfSyncDockerfiles, invalidTemplates); + } + + if (outOfSyncDockerfiles.Any() || invalidTemplates.Any()) + { + if (outOfSyncDockerfiles.Any()) + { + string dockerfileList = string.Join(Environment.NewLine, outOfSyncDockerfiles); + Logger.WriteError($"Dockerfiles out of sync with templates:{Environment.NewLine}{dockerfileList}"); + } + + if (invalidTemplates.Any()) + { + string templateList = string.Join(Environment.NewLine, invalidTemplates); + Logger.WriteError($"Invalid Templates:{Environment.NewLine}{templateList}"); + } + + _environmentService.Exit(1); + } + } + + private async Task GenerateDockerfileAsync(string template, PlatformInfo platform, List outOfSyncDockerfiles, List invalidTemplates) + { + try + { + IDocument document = Document.CreateDefault(template, _config).DocumentOrThrow; + string generatedDockerfile = document.Render(Context.CreateBuiltin(GetSymbols(platform))); + + string currentDockerfile = File.Exists(platform.DockerfilePath) ? + await File.ReadAllTextAsync(platform.DockerfilePath) : string.Empty; + if (currentDockerfile == generatedDockerfile) + { + Logger.WriteMessage("Dockerfile in sync with template"); + } + else if (Options.Validate) + { + Logger.WriteError("Dockerfile out of sync with template"); + outOfSyncDockerfiles.Add(platform.DockerfilePath); + } + else + { + if (Options.IsVerbose) + { + Logger.WriteMessage($"Generated Dockerfile:{Environment.NewLine}{generatedDockerfile}"); + } + + if (!Options.IsDryRun) + { + await File.WriteAllTextAsync(platform.DockerfilePath, generatedDockerfile); + Logger.WriteMessage($"Updated '{platform.DockerfilePath}'"); + } + } + } + catch (ParseException e) + { + Logger.WriteError($"Error: {e}{Environment.NewLine}Invalid Syntax:{Environment.NewLine}{template.Substring(e.LocationStart)}"); + invalidTemplates.Add(platform.DockerfileTemplate); + } + } + + private static string GetOsArchHyphenatedName(PlatformInfo platform) + { + string osName; + if (platform.BaseOsVersion.Contains("nanoserver")) + { + string version = platform.BaseOsVersion.Split('-')[1]; + osName = $"NanoServer-{version}"; + } + else + { + osName = platform.GetOSDisplayName().Replace(' ', '-'); + } + + string archName = platform.Model.Architecture != Architecture.AMD64 ? $"-{platform.Model.Architecture.GetDisplayName()}" : string.Empty; + + return osName + archName; + } + + public IReadOnlyDictionary GetSymbols(PlatformInfo platform) + { + string versionedArch = platform.Model.Architecture.GetDisplayName(platform.Model.Variant); + + return new Dictionary + { + ["ARCH_SHORT"] = platform.Model.Architecture.GetShortName(), + ["ARCH_NUPKG"] = platform.Model.Architecture.GetNupkgName(), + ["ARCH_VERSIONED"] = versionedArch, + ["ARCH_TAG_SUFFIX"] = platform.Model.Architecture != Architecture.AMD64 ? $"-{versionedArch}" : string.Empty, + ["OS_VERSION"] = platform.Model.OsVersion, + ["OS_VERSION_BASE"] = platform.BaseOsVersion, + ["OS_VERSION_NUMBER"] = Regex.Match(platform.Model.OsVersion, @"\d+.\d+").Value, + ["OS_ARCH_HYPHENATED"] = GetOsArchHyphenatedName(platform), + ["VARIABLES"] = Manifest.Model?.Variables?.ToDictionary(kvp => (Value)kvp.Key, kvp => (Value)kvp.Value) + }; + } + } +} diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesOptions.cs b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesOptions.cs new file mode 100644 index 00000000..58e88cfc --- /dev/null +++ b/src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateDockerfilesOptions.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.CommandLine; + +namespace Microsoft.DotNet.ImageBuilder.Commands +{ + public class GenerateDockerfilesOptions : ManifestOptions, IFilterableOptions + { + protected override string CommandHelp => "Generates the Dockerfiles from Cottle based templates (http://r3c.github.io/cottle/)"; + + public ManifestFilterOptions FilterOptions { get; } = new ManifestFilterOptions(); + + public bool Validate { get; set; } + + public GenerateDockerfilesOptions() : base() + { + } + + public override void DefineOptions(ArgumentSyntax syntax) + { + base.DefineOptions(syntax); + + FilterOptions.DefineOptions(syntax); + + bool validate = false; + syntax.DefineOption("validate", ref validate, "Validates the Dockerfiles and templates are in sync"); + Validate = validate; + } + } +} diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Microsoft.DotNet.ImageBuilder.csproj b/src/Microsoft.DotNet.ImageBuilder/src/Microsoft.DotNet.ImageBuilder.csproj index def3ec84..81d505c0 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Microsoft.DotNet.ImageBuilder.csproj +++ b/src/Microsoft.DotNet.ImageBuilder/src/Microsoft.DotNet.ImageBuilder.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Microsoft.DotNet.ImageBuilder/src/Models/Manifest/Platform.cs b/src/Microsoft.DotNet.ImageBuilder/src/Models/Manifest/Platform.cs index c525527e..01da4ddf 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/Models/Manifest/Platform.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/Models/Manifest/Platform.cs @@ -35,6 +35,11 @@ public class Platform [JsonProperty(Required = Required.Always)] public string Dockerfile { get; set; } + [Description( + "Relative path to the template the Dockerfile is generated from." + )] + public string DockerfileTemplate { get; set; } + [Description( "The generic name of the operating system associated with the image." )] diff --git a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/ModelExtensions.cs b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/ModelExtensions.cs index d194605f..5cae6987 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/ModelExtensions.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/ModelExtensions.cs @@ -34,6 +34,43 @@ public static string GetDisplayName(this Architecture architecture, string varia return displayName; } + public static string GetShortName(this Architecture architecture) + { + string shortName; + + switch (architecture) + { + case Architecture.AMD64: + shortName = "x64"; + break; + default: + shortName = architecture.ToString().ToLowerInvariant(); + break; + } + + return shortName; + } + + public static string GetNupkgName(this Architecture architecture) + { + string nupkgName; + + switch (architecture) + { + case Architecture.AMD64: + nupkgName = "x64"; + break; + case Architecture.ARM: + nupkgName = "arm32"; + break; + default: + nupkgName = architecture.ToString().ToLowerInvariant(); + break; + } + + return nupkgName; + } + public static string GetDockerName(this Architecture architecture) => architecture.ToString().ToLowerInvariant(); public static string GetDockerName(this OS os) => os.ToString().ToLowerInvariant(); @@ -109,6 +146,7 @@ private static void ValidateImage(Image image, string manifestDirectory) private static void ValidatePlatform(Platform platform, string manifestDirectory) { ValidateFileReference(platform.ResolveDockerfilePath(manifestDirectory), manifestDirectory); + ValidateFileReference(platform.DockerfileTemplate, manifestDirectory); } private static void ValidateUniqueTags(Repo repo) diff --git a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs index c49abdd6..adaafb01 100644 --- a/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs +++ b/src/Microsoft.DotNet.ImageBuilder/src/ViewModel/PlatformInfo.cs @@ -29,6 +29,7 @@ public class PlatformInfo public string BuildContextPath { get; private set; } public string DockerfilePath { get; private set; } public string DockerfilePathRelativeToManifest { get; private set; } + public string DockerfileTemplate { get; private set; } public string FinalStageFromImage { get; private set; } public IEnumerable ExternalFromImages { get; private set; } public IEnumerable InternalFromImages { get; private set; } @@ -57,6 +58,11 @@ public static PlatformInfo Create(Platform model, string fullRepoModelName, stri platformInfo.BuildContextPath = PathHelper.NormalizePath(Path.GetDirectoryName(dockerfileWithBaseDir)); platformInfo.DockerfilePathRelativeToManifest = PathHelper.TrimPath(baseDirectory, platformInfo.DockerfilePath); + if (model.DockerfileTemplate != null) + { + platformInfo.DockerfileTemplate = Path.Combine(baseDirectory, model.DockerfileTemplate); + } + platformInfo.Tags = model.Tags .Select(kvp => TagInfo.Create(kvp.Key, kvp.Value, repoName, variableHelper, platformInfo.BuildContextPath)) .ToArray(); diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/GenerateDockerfilesCommandTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/GenerateDockerfilesCommandTests.cs new file mode 100644 index 00000000..71f519aa --- /dev/null +++ b/src/Microsoft.DotNet.ImageBuilder/tests/GenerateDockerfilesCommandTests.cs @@ -0,0 +1,175 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Cottle; +using Microsoft.DotNet.ImageBuilder.Commands; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.Tests.Helpers; +using Moq; +using Newtonsoft.Json; +using Xunit; +using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ManifestHelper; + +namespace Microsoft.DotNet.ImageBuilder.Tests +{ + public class GenerateDockerfilesCommandTests + { + private const string DockerfilePath = "1.0/sdk/os/Dockerfile"; + private const string DefaultDockerfile = "FROM Base"; + private const string DockerfileTemplatePath = "Dockerfile.Template"; + private const string DefaultDockerfileTemplate = +@"FROM Repo:2.1-{{OS_VERSION_BASE}} +ENV TEST1 {{if OS_VERSION = ""buster-slim"":IfWorks}} +ENV TEST2 {{VARIABLES[""Variable1""]}}"; + private const string ExpectedDockerfile = +@"FROM Repo:2.1-buster +ENV TEST1 IfWorks +ENV TEST2 Value1"; + + private readonly Exception _exitException = new Exception(); + Mock _environmentServiceMock; + + public GenerateDockerfilesCommandTests() + { + } + + [Fact] + public async Task GenerateDockerfilesCommand_Canonical() + { + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateDockerfilesCommand command = InitializeCommand(tempFolderContext); + + await command.ExecuteAsync(); + + string generatedDockerfile = File.ReadAllText(Path.Combine(tempFolderContext.Path, DockerfilePath)); + Assert.Equal(ExpectedDockerfile.NormalizeLineEndings(generatedDockerfile), generatedDockerfile); + } + + [Fact] + public async Task GenerateDockerfilesCommand_InvalidTemplate() + { + string template = "FROM $REPO:2.1-{{if:}}"; + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateDockerfilesCommand command = InitializeCommand(tempFolderContext, template); + + Exception actualException = await Assert.ThrowsAsync(command.ExecuteAsync); + + Assert.Same(_exitException, actualException); + _environmentServiceMock.Verify(o => o.Exit(It.IsAny()), Times.Once); + } + + [Fact] + public async Task GenerateDockerfilesCommand_Validate_UpToDate() + { + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateDockerfilesCommand command = InitializeCommand(tempFolderContext, dockerfile:ExpectedDockerfile, validate: true); + + await command.ExecuteAsync(); + + _environmentServiceMock.Verify(o => o.Exit(It.IsAny()), Times.Never); + } + + [Fact] + public async Task GenerateDockerfilesCommand_Validate_OutOfSync() + { + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateDockerfilesCommand command = InitializeCommand(tempFolderContext, validate: true); + + Exception actualException = await Assert.ThrowsAsync(command.ExecuteAsync); + + Assert.Same(_exitException, actualException); + _environmentServiceMock.Verify(o => o.Exit(It.IsAny()), Times.Once); + // Validate Dockerfile remains unmodified + Assert.Equal(DefaultDockerfile, File.ReadAllText(Path.Combine(tempFolderContext.Path, DockerfilePath))); + } + + [Theory] + [InlineData("repo1:tag1", "ARCH_SHORT", "arm")] + [InlineData("repo1:tag1", "ARCH_NUPKG", "arm32")] + [InlineData("repo1:tag1", "ARCH_VERSIONED", "arm32v7")] + [InlineData("repo1:tag2", "ARCH_VERSIONED", "amd64")] + [InlineData("repo1:tag1", "ARCH_TAG_SUFFIX", "-arm32v7")] + [InlineData("repo1:tag2", "ARCH_TAG_SUFFIX", "")] + [InlineData("repo1:tag1", "OS_VERSION", "buster-slim")] + [InlineData("repo1:tag1", "OS_VERSION_BASE", "buster")] + [InlineData("repo1:tag1", "OS_VERSION_NUMBER", "")] + [InlineData("repo1:tag3", "OS_VERSION_NUMBER", "3.12")] + [InlineData("repo1:tag1", "OS_ARCH_HYPHENATED", "Debian-10-arm32")] + [InlineData("repo1:tag2", "OS_ARCH_HYPHENATED", "NanoServer-1903")] + [InlineData("repo1:tag3", "OS_ARCH_HYPHENATED", "Alpine-3.12")] + [InlineData("repo1:tag1", "Variable1", "Value1", true)] + public void GenerateDockerfilesCommand_SupportedSymbols(string tag, string symbol, string expectedValue, bool isVariable = false) + { + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + GenerateDockerfilesCommand command = InitializeCommand(tempFolderContext); + + IReadOnlyDictionary symbols = command.GetSymbols(command.Manifest.GetPlatformByTag(tag)); + + Value variableValue; + if (isVariable) + { + variableValue = symbols["VARIABLES"].Fields[symbol]; + } + else + { + variableValue = symbols[symbol]; + } + + Assert.Equal(expectedValue, variableValue); + } + + private GenerateDockerfilesCommand InitializeCommand( + TempFolderContext tempFolderContext, + string dockerfileTemplate = DefaultDockerfileTemplate, + string dockerfile = DefaultDockerfile, + bool validate = false) + { + DockerfileHelper.CreateFile(DockerfileTemplatePath, tempFolderContext, dockerfileTemplate); + DockerfileHelper.CreateFile(DockerfilePath, tempFolderContext, dockerfile); + + Manifest manifest = CreateManifest( + CreateRepo("repo1", + CreateImage( + CreatePlatform( + DockerfilePath, + new string[] { "tag1" }, + OS.Linux, + "buster-slim", + Architecture.ARM, + "v7", + dockerfileTemplatePath: DockerfileTemplatePath), + CreatePlatform( + DockerfilePath, + new string[] { "tag2" }, + OS.Windows, + "nanoserver-1903"), + CreatePlatform( + DockerfilePath, + new string[] { "tag3" }, + OS.Linux, + "alpine3.12"))) + ); + AddVariable(manifest, "Variable1", "Value1"); + + string manifestPath = Path.Combine(tempFolderContext.Path, "manifest.json"); + File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifest)); + + _environmentServiceMock = new Mock(); + _environmentServiceMock + .Setup(o => o.Exit(1)) + .Throws(_exitException); + + GenerateDockerfilesCommand command = new GenerateDockerfilesCommand(_environmentServiceMock.Object); + command.Options.Manifest = manifestPath; + command.Options.Validate = validate; + command.LoadManifest(); + + return command; + } + } +} diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/DockerfileHelper.cs b/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/DockerfileHelper.cs index 781e49f5..78ed08c9 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/DockerfileHelper.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/DockerfileHelper.cs @@ -10,10 +10,16 @@ public static class DockerfileHelper { public static string CreateDockerfile(string relativeDirectory, TempFolderContext context, string fromTag = "base") { - Directory.CreateDirectory(Path.Combine(context.Path, relativeDirectory)); - string dockerfileRelativePath = Path.Combine(relativeDirectory, "Dockerfile"); - File.WriteAllText(PathHelper.NormalizePath(Path.Combine(context.Path, dockerfileRelativePath)), $"FROM {fromTag}"); - return PathHelper.NormalizePath(dockerfileRelativePath); + string relativeDockerfilePath = PathHelper.NormalizePath(Path.Combine(relativeDirectory, "Dockerfile")); + CreateFile(relativeDockerfilePath, context, $"FROM {fromTag}"); + return relativeDockerfilePath; + } + + public static void CreateFile(string relativeFileName, TempFolderContext context, string content) + { + string fullFilePath = Path.Combine(context.Path, relativeFileName); + Directory.CreateDirectory(Directory.GetParent(fullFilePath).FullName); + File.WriteAllText(fullFilePath, content); } } } diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/ManifestHelper.cs b/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/ManifestHelper.cs index fd96ac2a..83e500be 100644 --- a/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/ManifestHelper.cs +++ b/src/Microsoft.DotNet.ImageBuilder/tests/Helpers/ManifestHelper.cs @@ -50,20 +50,38 @@ public static Image CreateImage(Platform[] platforms, IDictionary s } public static Platform CreatePlatform( - string dockerfilePath, string[] tags, OS os = OS.Linux, string osVersion = "disco", Architecture architecture = Architecture.AMD64, - CustomBuildLegGrouping[] customBuildLegGroupings = null) + string dockerfilePath, + string[] tags, + OS os = OS.Linux, + string osVersion = "disco", + Architecture architecture = Architecture.AMD64, + string variant = null, + CustomBuildLegGrouping[] customBuildLegGroupings = null, + string dockerfileTemplatePath = null) { return new Platform { Dockerfile = dockerfilePath, + DockerfileTemplate = dockerfileTemplatePath, OsVersion = osVersion, OS = os, Tags = tags.ToDictionary(tag => tag, tag => new Tag()), Architecture = architecture, + Variant = variant, CustomBuildLegGrouping = customBuildLegGroupings ?? Array.Empty() }; } + public static void AddVariable(Manifest manifest, string name, string value) + { + if (manifest.Variables == null) + { + manifest.Variables = new Dictionary(); + } + + manifest.Variables.Add(name, value); + } + public static IManifestOptionsInfo GetManifestOptions(string manifestPath) { Mock manifestOptionsMock = new Mock(); diff --git a/src/Microsoft.DotNet.ImageBuilder/tests/ModelExtensionsTests.cs b/src/Microsoft.DotNet.ImageBuilder/tests/ModelExtensionsTests.cs new file mode 100644 index 00000000..c1fcb66a --- /dev/null +++ b/src/Microsoft.DotNet.ImageBuilder/tests/ModelExtensionsTests.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ViewModel; +using Xunit; + +namespace Microsoft.DotNet.ImageBuilder.Tests +{ + public class ModelExtensionsTests + { + [Theory] + [InlineData(Architecture.AMD64, "amd64")] + [InlineData(Architecture.ARM, "arm32")] + [InlineData(Architecture.ARM64, "arm64")] + [InlineData(Architecture.AMD64, "amd64v8", "v8")] + [InlineData(Architecture.ARM, "arm32v7", "V7")] + public void Architecture_GetDisplayName(Architecture architecture, string expectedDisplayName, string variant = null) + { + Assert.Equal(expectedDisplayName, architecture.GetDisplayName(variant)); + } + + [Theory] + [InlineData(Architecture.AMD64, "x64")] + [InlineData(Architecture.ARM, "arm")] + [InlineData(Architecture.ARM64, "arm64")] + public void Architecture_GetShortName(Architecture architecture, string expectedShortName) + { + Assert.Equal(expectedShortName, architecture.GetShortName()); + } + + [Theory] + [InlineData(Architecture.AMD64, "x64")] + [InlineData(Architecture.ARM, "arm32")] + [InlineData(Architecture.ARM64, "arm64")] + public void Architecture_GetNupkgName(Architecture architecture, string expectedNupkgName) + { + Assert.Equal(expectedNupkgName, architecture.GetNupkgName()); + } + } +}