diff --git a/Documentation/ArcadeSdk.md b/Documentation/ArcadeSdk.md index 72edf5f55da..c6f0f88b2ec 100644 --- a/Documentation/ArcadeSdk.md +++ b/Documentation/ArcadeSdk.md @@ -212,7 +212,6 @@ optimizations by setting 'RestoreUsingNuGetTargets' to false. CoreFx does not use the default build projects in its repo - [example](https://github.com/dotnet/corefx/blob/66392f577c7852092f668876822b6385bcafbd44/eng/Build.props). - ### /eng/Versions.props: A single file listing component versions and used tools The file is present in the repo and defines versions of all dependencies used in the repository, the NuGet feeds they should be restored from and the version of the components produced by the repo build. @@ -293,7 +292,7 @@ Targets executed in a step right after the solution is built. Targets executed in a step right after artifacts has been signed. -### /global.json, /NuGet.config +### /global.json `/global.json` file is present and specifies the version of the dotnet and `Microsoft.DotNet.Arcade.Sdk` SDKs. @@ -348,6 +347,60 @@ The version of `RoslynTools.MSBuild` package can be specified in `global.json` f If it is not specified the build script attempts to find `RoslynTools.MSBuild` version `{VSMajor}.{VSMinor}.0-alpha` where `VSMajor.VSMinor` is the value of `tools.vs.version`. +#### Example: Restoring multiple .NET Core Runtimes for running tests + +In /global.json, specify a `runtimes` section and list the [shared runtime versions](https://dotnet.microsoft.com/download/dotnet-core) you want installed. + +Schema: + +```text +{ + "tools": { + "dotnet": "", // define CLI SDK version + "runtimes": { // optional runtimes section + "": [ "", ..., "" ], + ..., + "/": [ "", ..., "" ] + } + } +} +``` + +`` - One of the supported "runtime" values for the [dotnet-install](https://github.com/dotnet/cli/blob/dddac220ba5b6994e297752bebd9acffa3e72342/scripts/obtain/dotnet-install.ps1#L43) script. + +`` - Optionally include `/` when defining the runtime to specify an explicit architecture where "architecture" is one of the supported values for the [dotnet-install](https://github.com/dotnet/cli/blob/dddac220ba5b6994e297752bebd9acffa3e72342/scripts/obtain/dotnet-install.ps1#L32) script. Defaults to "auto" if not specified. + +```json +{ + "tools": { + "dotnet": "3.0.100-preview3-010431", + "runtimes": { + "dotnet/x64": [ "2.1.7" ], + "aspnetcore/x64": [ "3.0.0-build-20190219.1" ] + } + } +} +``` + +You may also use any of the properties defined in `eng/Versions.props` to define a version. + +Example + +```json +{ + "tools": { + "dotnet": "3.0.100-preview3-010431", + "runtimes": { + "dotnet/x64": [ "2.1.7", "$(MicrosoftNetCoreAppVersion)" ] + } + } +} +``` + +Note: defining `runtimes` in your global.json will signal to Arcade to install a local version of the SDK for the runtimes to use rather than depending on a matching global SDK. + +### /NuGet.config + `/NuGet.config` file is present and specifies the MyGet feed to retrieve Arcade SDK from like so: ```xml @@ -521,6 +574,7 @@ folder "InstallDir:MSBuild\Microsoft\VisualStudio\Managed" **NOTE:** By defining `VisualStudioInsertionComponent` in your project you are implicitly opting-in to having all of the assemblies included in that package marked for `NGEN`. If this is not something you want for a given component you may add `false`. example: + ```xml diff --git a/eng/Versions.props b/eng/Versions.props index dda00176554..6e14caa3fa7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -34,19 +34,24 @@ 5.3.0.1 2.3.0 9.0.1 + 4.6.0-preview4.19202.2 4.4.0 4.9.0-rtm.5658 0.32.0 2.2.127 3.0.0 + 4.5.0 1.3.1 4.0.0 4.3.0 4.5.0 4.3.0 4.5.1 + 4.4.0 1.4.2 + 4.5.0 4.5.0 + 4.5.1 4.4.0 8.5.0 2.4.1-pre.build.4059 diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd new file mode 100644 index 00000000000..b1c2642e76f --- /dev/null +++ b/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 new file mode 100644 index 00000000000..5987943fd6f --- /dev/null +++ b/eng/common/dotnet-install.ps1 @@ -0,0 +1,22 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [string] $architecture = "", + [string] $version = "Latest", + [string] $runtime = "dotnet" +) + +. $PSScriptRoot\tools.ps1 + +try { + $dotnetRoot = Join-Path $RepoRoot ".dotnet" + InstallDotNet $dotnetRoot $version $architecture $runtime $true +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh new file mode 100755 index 00000000000..c3072c958af --- /dev/null +++ b/eng/common/dotnet-install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +version='Latest' +architecture='' +runtime='dotnet' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + shift +done + +. "$scriptroot/tools.sh" +dotnetRoot="$repo_root/.dotnet" +InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { + local exit_code=$? + echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 09794dff837..d86eef1e3a7 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -108,7 +108,7 @@ function InitializeDotNetCli([bool]$install) { } # Find the first path on %PATH% that contains the dotnet.exe - if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { + if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent @@ -119,7 +119,7 @@ function InitializeDotNetCli([bool]$install) { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" @@ -152,7 +152,7 @@ function InitializeDotNetCli([bool]$install) { } function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = "$dotnetRoot\dotnet-install.ps1" + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript @@ -162,9 +162,21 @@ function GetDotNetInstallScript([string] $dotnetRoot) { } function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { + InstallDotNet $dotnetRoot $version $architecture +} + +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot - $archArg = if ($architecture) { $architecture } else { "" } - & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg + $installParameters = @{ + Version = $version + InstallDir = $dotnetRoot + } + + if ($architecture) { $installParameters.Architecture = $architecture } + if ($runtime) { $installParameters.Runtime = $runtime } + if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + + & $installScript @installParameters if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode @@ -429,6 +441,7 @@ function InitializeToolset() { $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '' | Set-Content $proj + MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile $path = Get-Content $toolsetLocationFile -TotalCount 1 @@ -522,6 +535,8 @@ $ToolsDir = Join-Path $RepoRoot ".tools" $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json +# true if global.json contains a "runtimes" section +$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } Create-Directory $ToolsetDir Create-Directory $TempDir @@ -534,4 +549,4 @@ if ($ci) { $env:TEMP = $TempDir $env:TMP = $TempDir -} +} \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh old mode 100644 new mode 100755 index 59f47c5fa90..9dc565e2961 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -101,7 +101,7 @@ function InitializeDotNetCli { fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -115,10 +115,11 @@ function InitializeDotNetCli { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else dotnet_root="$repo_root/.dotnet" + export DOTNET_INSTALL_DIR="$dotnet_root" if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then @@ -149,16 +150,34 @@ function InitializeDotNetCli { function InstallDotNetSdk { local root=$1 local version=$2 + local architecture="" + if [[ $# == 3 ]]; then + architecture=$3 + fi + InstallDotNet "$root" "$version" $architecture +} +function InstallDotNet { + local root=$1 + local version=$2 + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local arch_arg="" - if [[ $# == 3 ]]; then - arch_arg="--architecture $3" + local archArg='' + if [[ "$#" -ge "3" ]]; then + archArg="--architecture $3" + fi + local runtimeArg='' + if [[ "$#" -ge "4" ]]; then + runtimeArg="--runtime $4" fi - bash "$install_script" --version $version --install-dir "$root" $arch_arg || { + local skipNonVersionedFilesArg="" + if [[ "$#" -ge "5" ]]; then + skipNonVersionedFilesArg="--skip-non-versioned-files" + fi + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code @@ -323,6 +342,12 @@ log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" +# determine if global.json contains a "runtimes" entry +global_json_has_runtimes=false +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +if [[ -n "$dotnetlocal_key" ]]; then + global_json_has_runtimes=true +fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then @@ -337,4 +362,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi +fi \ No newline at end of file diff --git a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj index f6f6c07254a..40eee44bdc9 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj +++ b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj @@ -2,7 +2,8 @@ net472;netcoreapp2.1 - + preview + true false true @@ -16,6 +17,7 @@ false false <_GeneratedVersionFilePath>$(IntermediateOutputPath)DefaultVersions.Generated.props + 3021;NU5105 @@ -27,6 +29,16 @@ + + + + + + + + + + @@ -45,6 +57,7 @@ + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs b/src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs new file mode 100644 index 00000000000..cafaea72f55 --- /dev/null +++ b/src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs @@ -0,0 +1,148 @@ +using Microsoft.Build.Evaluation; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Versioning; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; + +namespace Microsoft.DotNet.Arcade.Sdk +{ +#if NET472 + [LoadInSeparateAppDomain] + public class InstallDotNetCore : AppDomainIsolatedTask + { + static InstallDotNetCore() => AssemblyResolution.Initialize(); +#else + public class InstallDotNetCore : Task + { +#endif + public string VersionsPropsPath { get; set; } + + [Required] + public string DotNetInstallScript { get; set; } + [Required] + public string GlobalJsonPath { get; set; } + + public override bool Execute() + { + if (!File.Exists(GlobalJsonPath)) + { + Log.LogWarning($"Unable to find global.json file '{GlobalJsonPath} exiting"); + return true; + } + if (!File.Exists(DotNetInstallScript)) + { + Log.LogError($"Unable to find dotnet install script '{DotNetInstallScript} exiting"); + return !Log.HasLoggedErrors; + } + + var jsonContent = File.ReadAllText(GlobalJsonPath); + var bytes = Encoding.UTF8.GetBytes(jsonContent); + + using (JsonDocument jsonDocument = JsonDocument.Parse(bytes)) + { + if (jsonDocument.RootElement.TryGetProperty("tools", out JsonElement toolsElement)) + { + if (toolsElement.TryGetProperty("runtimes", out JsonElement dotnetLocalElement)) + { + var runtimeItems = new Dictionary>>(); + foreach (var runtime in dotnetLocalElement.EnumerateObject()) + { + var items = GetItemsFromJsonElementArray(runtime, out string runtimeName); + runtimeItems.Add(runtimeName, items); + } + if (runtimeItems.Count > 0) + { + System.Linq.ILookup properties = null; + // Only load Versions.props if there's a need to look for a version identifier (ie, there's a value listed that's not a parsable version). + if (runtimeItems.SelectMany(r => r.Value).Select(r => r.Key).FirstOrDefault(f => !SemanticVersion.TryParse(f, out SemanticVersion version)) != null) + { + if (!File.Exists(VersionsPropsPath)) + { + Log.LogError($"Unable to find translation file {VersionsPropsPath}"); + return !Log.HasLoggedErrors; + } + else + { + var proj = Project.FromFile(VersionsPropsPath, new Build.Definition.ProjectOptions()); + properties = proj.AllEvaluatedProperties.ToLookup(p => p.Name); + } + } + + foreach(var runtimeItem in runtimeItems) + { + foreach(var item in runtimeItem.Value) + { + SemanticVersion version = null; + // Try to parse version + if (!SemanticVersion.TryParse(item.Key, out version)) + { + var propertyName = item.Key.Trim('$', '(', ')'); + + // Unable to parse version, try to find the corresponding identifer from the MSBuild loaded MSBuild properties + string evaluatedValue = properties[propertyName].First().EvaluatedValue; + if (!SemanticVersion.TryParse(evaluatedValue, out version)) + { + Log.LogError($"Unable to parse '{item.Key}' from properties defined in '{VersionsPropsPath}'"); + } + } + + if(version != null) + { + string arguments = $"-runtime \"{runtimeItem.Key}\" -version \"{version.ToNormalizedString()}\""; + if (!string.IsNullOrWhiteSpace(item.Value)) + { + arguments += $" -architecture {item.Value}"; + } + Log.LogMessage(MessageImportance.Low, $"Executing: {DotNetInstallScript} {arguments}"); + var process = Process.Start(new ProcessStartInfo() + { + FileName = DotNetInstallScript, + Arguments = arguments, + UseShellExecute = false + }); + process.WaitForExit(); + if(process.ExitCode != 0) + { + Log.LogError("dotnet-install failed"); + } + } + } + } + } + } + } + } + return !Log.HasLoggedErrors; + } + + /* + * Parses a json token of this format + * { (runtime): [(version), ..., (version)] } + * or this format + * { (runtime/architecture): [(version), ..., (version)] } + */ + private IEnumerable> GetItemsFromJsonElementArray(JsonProperty token, out string runtime) + { + var items = new List>(); + + runtime = token.Name; + string architecture = string.Empty; + if(runtime.Contains('/')) + { + var parts = runtime.Split(new char[] { '/' }, 2); + runtime = parts[0]; + architecture = parts[1]; + } + foreach (var version in token.Value.EnumerateArray()) + { + items.Add(new KeyValuePair(version.GetString(), architecture)); + } + return items.ToArray(); + } + } +} diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/InstallDotNetCore.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/InstallDotNetCore.targets new file mode 100644 index 00000000000..5dd08f5a0ba --- /dev/null +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/InstallDotNetCore.targets @@ -0,0 +1,18 @@ + + + + + + + <_DotNetInstallScript>$(RepositoryEngineeringDir)common\dotnet-install.cmd + <_DotNetInstallScript Condition="'$(OS)' != 'Windows_NT'">$(RepositoryEngineeringDir)common\dotnet-install.sh + + + + + + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj index 030883744e3..2957e78213f 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj @@ -57,5 +57,7 @@ - + + +