Multi framework (#2343)

* PR feedback: move install dotnet out of sdk restore, into tools restore

* Fix comment

* PR feedback, remove unneeded variable, fix typos

Documentation/ArcadeSdk.md

@@ -212,7 +212,6 @@ optimizations by setting 'RestoreUsingNuGetTargets' to false.
 
 CoreFx does not use the default build projects in its repo - [example](https://github.com/dotnet/corefx/blob/66392f577c7852092f668876822b6385bcafbd44/eng/Build.props).
 
-
 ### /eng/Versions.props: A single file listing component versions and used tools
 
 The file is present in the repo and defines versions of all dependencies used in the repository, the NuGet feeds they should be restored from and the version of the components produced by the repo build.
@@ -293,7 +292,7 @@ Targets executed in a step right after the solution is built.
 
 Targets executed in a step right after artifacts has been signed.
 
-### /global.json, /NuGet.config
+### /global.json
 
 `/global.json` file is present and specifies the version of the dotnet and `Microsoft.DotNet.Arcade.Sdk` SDKs.
 
@@ -348,6 +347,60 @@ The version of `RoslynTools.MSBuild` package can be specified in `global.json` f
 
 If it is not specified the build script attempts to find `RoslynTools.MSBuild` version `{VSMajor}.{VSMinor}.0-alpha` where `VSMajor.VSMinor` is the value of `tools.vs.version`.
 
+#### Example: Restoring multiple .NET Core Runtimes for running tests
+
+In /global.json, specify a `runtimes` section and list the [shared runtime versions](https://dotnet.microsoft.com/download/dotnet-core) you want installed.
+
+Schema:
+
+```text
+{
+  "tools": {
+    "dotnet": "<version>",                                           // define CLI SDK version
+    "runtimes": {                                                    // optional runtimes section
+      "<runtime>": [ "<version>", ..., "<version>" ],
+      ...,
+      "<runtime>/<architecture>": [ "<version>", ..., "<version>" ]
+    }
+  }
+}
+```
+
+`<runtime>` - One of the supported "runtime" values for the [dotnet-install](https://github.com/dotnet/cli/blob/dddac220ba5b6994e297752bebd9acffa3e72342/scripts/obtain/dotnet-install.ps1#L43) script.
+
+`<architecture>` - Optionally include `/<architecture>` when defining the runtime to specify an explicit architecture where "architecture" is one of the supported values for the [dotnet-install](https://github.com/dotnet/cli/blob/dddac220ba5b6994e297752bebd9acffa3e72342/scripts/obtain/dotnet-install.ps1#L32) script.  Defaults to "auto" if not specified.
+
+```json
+{
+  "tools": {
+    "dotnet": "3.0.100-preview3-010431",
+    "runtimes": {
+      "dotnet/x64": [ "2.1.7" ],
+      "aspnetcore/x64": [ "3.0.0-build-20190219.1" ]
+    }
+  }
+}
+```
+
+You may also use any of the properties defined in `eng/Versions.props` to define a version.
+
+Example
+
+```json
+{
+  "tools": {
+    "dotnet": "3.0.100-preview3-010431",
+    "runtimes": {
+      "dotnet/x64": [ "2.1.7", "$(MicrosoftNetCoreAppVersion)" ]
+    }
+  }
+}
+```
+
+Note: defining `runtimes` in your global.json will signal to Arcade to install a local version of the SDK for the runtimes to use rather than depending on a matching global SDK.
+
+### /NuGet.config
+
 `/NuGet.config` file is present and specifies the MyGet feed to retrieve Arcade SDK from like so:
 
 ```xml
@@ -521,6 +574,7 @@ folder "InstallDir:MSBuild\Microsoft\VisualStudio\Managed"
 **NOTE:** By defining `VisualStudioInsertionComponent` in your project you are implicitly opting-in to having all of the assemblies included in that package marked for `NGEN`.  If this is not something you want for a given component you may add `<Ngen>false</Ngen>`.
 
 example:
+
 ```xml
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>

eng/Versions.props

@@ -34,19 +34,24 @@
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
     <McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
     <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+    <MicrosoftBclJsonSourcesVersion>4.6.0-preview4.19202.2</MicrosoftBclJsonSourcesVersion>
     <NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
     <NuGetVersion>4.9.0-rtm.5658</NuGetVersion>
     <OctokitVersion>0.32.0</OctokitVersion>
     <DotNetSleetLibVersion>2.2.127</DotNetSleetLibVersion>
     <SwashbuckleAspNetCoreSwaggerVersion>3.0.0</SwashbuckleAspNetCoreSwaggerVersion>
+    <SystemBuffersVersion>4.5.0</SystemBuffersVersion>
     <SystemCollectionsImmutableVersion>1.3.1</SystemCollectionsImmutableVersion>
     <SystemDiagnosticsTraceSourceVersion>4.0.0</SystemDiagnosticsTraceSourceVersion>
     <SystemIOCompressionVersion>4.3.0</SystemIOCompressionVersion>
     <SystemIOPackagingVersion>4.5.0</SystemIOPackagingVersion>
     <SystemIOFileSystemPrimitivesVersion>4.3.0</SystemIOFileSystemPrimitivesVersion>
     <SystemMemoryVersion>4.5.1</SystemMemoryVersion>
+    <SystemNumericsVectorsVersion>4.4.0</SystemNumericsVectorsVersion>
     <SystemReflectionMetadataVersion>1.4.2</SystemReflectionMetadataVersion>
+    <SystemRuntimeCompilerServicesUnsafeVersion>4.5.0</SystemRuntimeCompilerServicesUnsafeVersion>
     <SystemTextEncodingsWebVersion>4.5.0</SystemTextEncodingsWebVersion>
+    <SystemThreadingTasksExtensionVersion>4.5.1</SystemThreadingTasksExtensionVersion>
     <SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
     <WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
     <XUnitVersion>2.4.1-pre.build.4059</XUnitVersion>

eng/common/dotnet-install.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*"

eng/common/dotnet-install.ps1

@@ -0,0 +1,22 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+  [string] $verbosity = "minimal",
+  [string] $architecture = "",
+  [string] $version = "Latest",
+  [string] $runtime = "dotnet"
+)
+
+. $PSScriptRoot\tools.ps1
+
+try {
+  $dotnetRoot = Join-Path $RepoRoot ".dotnet"
+  InstallDotNet $dotnetRoot $version $architecture $runtime $true
+} 
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}
+
+ExitWithExitCode 0

eng/common/dotnet-install.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+version='Latest'
+architecture=''
+runtime='dotnet'
+while [[ $# > 0 ]]; do
+  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  case "$opt" in
+    -version|-v)
+      shift
+      version="$1"
+      ;;
+    -architecture|-a)
+      shift
+      architecture="$1"
+      ;;
+    -runtime|-r)
+      shift
+      runtime="$1"
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+. "$scriptroot/tools.sh"
+dotnetRoot="$repo_root/.dotnet"
+InstallDotNet $dotnetRoot $version "$architecture" $runtime true || {
+  local exit_code=$?
+  echo "dotnet-install.sh failed (exit code '$exit_code')." >&2
+  ExitWithExitCode $exit_code
+}
+
+ExitWithExitCode 0

eng/common/tools.ps1

@@ -108,7 +108,7 @@ function InitializeDotNetCli([bool]$install) {
   }
 
   # Find the first path on %PATH% that contains the dotnet.exe
-  if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
+  if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
     $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue
     if ($dotnetCmd -ne $null) {
       $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
@@ -119,7 +119,7 @@ function InitializeDotNetCli([bool]$install) {
 
   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
-  if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+  if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
     $dotnetRoot = $env:DOTNET_INSTALL_DIR
   } else {
     $dotnetRoot = Join-Path $RepoRoot ".dotnet"
@@ -152,7 +152,7 @@ function InitializeDotNetCli([bool]$install) {
 }
 
 function GetDotNetInstallScript([string] $dotnetRoot) {
-  $installScript = "$dotnetRoot\dotnet-install.ps1"
+  $installScript = Join-Path $dotnetRoot "dotnet-install.ps1"
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
     Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript
@@ -162,9 +162,21 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
 }
 
 function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") {
+  InstallDotNet $dotnetRoot $version $architecture
+}
+
+function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) {  $installScript = GetDotNetInstallScript $dotnetRoot
   $installScript = GetDotNetInstallScript $dotnetRoot
-  $archArg = if ($architecture) { $architecture } else { "<auto>" }
-  & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg
+  $installParameters = @{
+    Version = $version
+    InstallDir = $dotnetRoot
+  }
+
+  if ($architecture) { $installParameters.Architecture = $architecture }
+  if ($runtime) { $installParameters.Runtime = $runtime }
+  if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+
+  & $installScript @installParameters
   if ($lastExitCode -ne 0) {
     Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red
     ExitWithExitCode $lastExitCode
@@ -429,6 +441,7 @@ function InitializeToolset() {
   $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" }
 
   '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
+
   MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
 
   $path = Get-Content $toolsetLocationFile -TotalCount 1
@@ -522,6 +535,8 @@ $ToolsDir = Join-Path $RepoRoot ".tools"
 $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration
 $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration
 $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json
+# true if global.json contains a "runtimes" section
+$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
 
 Create-Directory $ToolsetDir
 Create-Directory $TempDir
@@ -534,4 +549,4 @@ if ($ci) {
 
   $env:TEMP = $TempDir
   $env:TMP = $TempDir
-}
+}

eng/common/tools.sh

@@ -101,7 +101,7 @@ function InitializeDotNetCli {
   fi
 
   # Find the first path on $PATH that contains the dotnet.exe
-  if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
+  if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
     local dotnet_path=`command -v dotnet`
     if [[ -n "$dotnet_path" ]]; then
       ResolvePath "$dotnet_path"
@@ -115,10 +115,11 @@ function InitializeDotNetCli {
 
   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
-  if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+  if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
     dotnet_root="$DOTNET_INSTALL_DIR"
   else
     dotnet_root="$repo_root/.dotnet"
+
     export DOTNET_INSTALL_DIR="$dotnet_root"
 
     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
@@ -149,16 +150,34 @@ function InitializeDotNetCli {
 function InstallDotNetSdk {
   local root=$1
   local version=$2
+  local architecture=""
+  if [[ $# == 3 ]]; then
+    architecture=$3
+  fi
+  InstallDotNet "$root" "$version" $architecture
+}
 
+function InstallDotNet {
+  local root=$1
+  local version=$2
+
   GetDotNetInstallScript "$root"
   local install_script=$_GetDotNetInstallScript
 
-  local arch_arg=""
-  if [[ $# == 3 ]]; then
-    arch_arg="--architecture $3"
+  local archArg=''
+  if [[ "$#" -ge "3" ]]; then
+    archArg="--architecture $3"
+  fi
+  local runtimeArg=''
+  if [[ "$#" -ge "4" ]]; then
+    runtimeArg="--runtime $4"
   fi
 
-  bash "$install_script" --version $version --install-dir "$root" $arch_arg || {
+  local skipNonVersionedFilesArg=""
+  if [[ "$#" -ge "5" ]]; then
+    skipNonVersionedFilesArg="--skip-non-versioned-files"
+  fi
+  bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
     local exit_code=$?
     echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2
     ExitWithExitCode $exit_code
@@ -323,6 +342,12 @@ log_dir="$artifacts_dir/log/$configuration"
 temp_dir="$artifacts_dir/tmp/$configuration"
 
 global_json_file="$repo_root/global.json"
+# determine if global.json contains a "runtimes" entry
+global_json_has_runtimes=false
+dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true
+if [[ -n "$dotnetlocal_key" ]]; then
+  global_json_has_runtimes=true
+fi
 
 # HOME may not be defined in some scenarios, but it is required by NuGet
 if [[ -z $HOME ]]; then
@@ -337,4 +362,4 @@ mkdir -p "$log_dir"
 if [[ $ci == true ]]; then
   export TEMP="$temp_dir"
   export TMP="$temp_dir"
-fi
+fi

src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj

@@ -2,7 +2,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFrameworks>net472;netcoreapp2.1</TargetFrameworks>
-
+    <LangVersion>preview</LangVersion>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
 
     <IsPackable>true</IsPackable>
@@ -16,6 +17,7 @@
     <EnableDefaultNoneItems>false</EnableDefaultNoneItems>
     <EnableGeneratedPackageContent>false</EnableGeneratedPackageContent>
     <_GeneratedVersionFilePath>$(IntermediateOutputPath)DefaultVersions.Generated.props</_GeneratedVersionFilePath>
+    <NoWarn>3021;NU5105</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>
@@ -27,6 +29,16 @@
     <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
   </ItemGroup>
 
+  <!-- Required for compiling "InstallDotNetCore.cs" -->
+  <ItemGroup Condition="'$(DotNetBuildFromSource)' != 'true'">
+    <PackageReference Include="Microsoft.Bcl.Json.Sources" Version="$(MicrosoftBclJsonSourcesVersion)" PrivateAssets="All" />
+    <PackageReference Include="System.Buffers" Version="$(SystemBuffersVersion)" />
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
+    <PackageReference Include="System.Numerics.Vectors" Version="$(SystemNumericsVectorsVersion)" />
+    <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="$(SystemRuntimeCompilerServicesUnsafeVersion)" />
+    <PackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionVersion)" />
+  </ItemGroup>
+
   <ItemGroup>
     <InternalsVisibleTo Include="Microsoft.DotNet.Arcade.Sdk.Tests" />
   </ItemGroup>
@@ -45,6 +57,7 @@
 
   <ItemGroup>
     <Compile Include="..\Common\AssemblyResolution.cs" Link="src\AssemblyResolution.cs" />
+    <Compile Remove="src\InstallDotNetCore.cs" Condition="'$(DotNetBuildFromSource)' == 'true'" />
   </ItemGroup>
 
   <Target Name="_GenerateSdkVersionFile" Inputs="$(MSBuildAllProjects)" Outputs="$(_GeneratedVersionFilePath)" BeforeTargets="GenerateNuSpec">