From 427a251a161c837b4186c0d799cc2dad01800542 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Thu, 25 Apr 2019 14:51:01 -0700 Subject: [PATCH 01/10] Reset files --- azure-pipelines.yml | 24 ++++++--- eng/Versions.props | 2 + eng/build.ps1 | 4 -- eng/build.sh | 5 +- eng/common/PublishToPackageFeed.proj | 28 +++++++--- eng/common/build.ps1 | 1 + eng/common/build.sh | 1 + eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- eng/common/dotnet-install.cmd | 2 + eng/common/dotnet-install.ps1 | 22 ++++++++ eng/common/dotnet-install.sh | 49 ++++++++++++++++++ eng/common/generate-graph-files.ps1 | 2 +- eng/common/internal/Tools.csproj | 8 ++- eng/common/templates/job/job.yml | 7 ++- .../templates/phases/publish-build-assets.yml | 2 + eng/common/templates/steps/helix-publish.yml | 51 ------------------- eng/common/templates/steps/send-to-helix.yml | 5 +- eng/common/tools.ps1 | 35 +++++++++++-- eng/common/tools.sh | 44 +++++++++++++--- global.json | 5 +- 21 files changed, 207 insertions(+), 94 deletions(-) mode change 100644 => 100755 eng/common/build.sh mode change 100644 => 100755 eng/common/darc-init.sh create mode 100644 eng/common/dotnet-install.cmd create mode 100644 eng/common/dotnet-install.ps1 create mode 100755 eng/common/dotnet-install.sh delete mode 100644 eng/common/templates/steps/helix-publish.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 79a47317c63c6..ff621063bf0fc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,6 +47,8 @@ jobs: timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: eng/cibuild.cmd -configuration $(_configuration) -prepareMachine -testDesktop -$(_testKind) -procdump displayName: Build and Test @@ -66,7 +68,7 @@ jobs: ArtifactName: 'Windows Desktop $(_configuration) $(_testKind)' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - job: Windows_Desktop_Spanish_Unit_Tests pool: @@ -75,6 +77,8 @@ jobs: timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: eng/cibuild.cmd -configuration Debug -prepareMachine -testDesktop displayName: Build and Test @@ -93,7 +97,7 @@ jobs: ArtifactName: 'Windows Desktop Spanish' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - job: Windows_CoreClr_Unit_Tests pool: @@ -109,6 +113,8 @@ jobs: timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: eng/cibuild.cmd -configuration $(_configuration) -prepareMachine -msbuildEngine:dotnet -testCoreClr displayName: Build and Test @@ -128,7 +134,7 @@ jobs: ArtifactName: 'Windows CoreClr $(_configuration)' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - job: Windows_Determinism_Test pool: @@ -136,6 +142,8 @@ jobs: queue: BuildPool.Windows.10.Amd64.VS2017.Open timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: eng/test-determinism.cmd -configuration Debug displayName: Build - Validate determinism @@ -146,7 +154,7 @@ jobs: ArtifactName: 'Build Determinism Files' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - job: Windows_Correctness_Test pool: @@ -154,6 +162,8 @@ jobs: queue: BuildPool.Windows.10.Amd64.VS2017.Open timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: eng/test-build-correctness.cmd -configuration Release displayName: Build - Validate correctness @@ -164,7 +174,7 @@ jobs: ArtifactName: 'Build Correctness Files' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - task: PublishBuildArtifacts@1 displayName: Publish Artifact Packages @@ -199,6 +209,8 @@ jobs: _configuration: Debug timeoutInMinutes: 90 steps: + - checkout: self + clean: true - script: ./eng/cibuild.sh --configuration $(_configuration) --prepareMachine $(_args) displayName: Build and Test - task: PublishTestResults@2 @@ -217,7 +229,7 @@ jobs: ArtifactName: 'Linux $(_name)' publishLocation: Container continueOnError: true - condition: not(succeeded()) + condition: always() - job: macOS_Test pool: diff --git a/eng/Versions.props b/eng/Versions.props index c7a26e881a8dc..a040cd47bf298 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -23,6 +23,7 @@ --> + 2.1.7 2.6.2-beta2 2.8.2 1.0.0-beta1-63310-01 @@ -277,6 +278,7 @@ + true true diff --git a/eng/build.ps1 b/eng/build.ps1 index dd47f54d95249..f68a4bec93c2c 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -583,10 +583,6 @@ try { $global:_DotNetInstallDir = Join-Path $RepoRoot ".dotnet" InstallDotNetSdk $global:_DotNetInstallDir $GlobalJson.tools.dotnet - - # Make sure a 2.1 runtime is installed so we can run our tests. Most of them still - # target netcoreapp2.1. - InstallDotNetSdk $global:_DotNetInstallDir "2.1.503" } if ($bootstrap) { diff --git a/eng/build.sh b/eng/build.sh index faaf32bd21933..1c30f04b67904 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -210,7 +210,7 @@ function BuildSolution { InitializeToolset local toolset_build_proj=$_InitializeToolset - + local bl="" if [[ "$binary_log" = true ]]; then bl="/bl:\"$log_dir/Build.binlog\"" @@ -280,9 +280,6 @@ function BuildSolution { InitializeDotNetCli $restore -# Make sure we have a 2.1 runtime available for running our tests -InstallDotNetSdk $_InitializeDotNetCli 2.1.503 - bootstrap_dir="" if [[ "$bootstrap" == true ]]; then MakeBootstrapBuild diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 25362ff060c11..e17f72644e3e3 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -1,11 +1,13 @@ - + + + netcoreapp2.1 @@ -41,6 +43,16 @@ https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json + https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json + PackageAssetsBasePath="$(PackageBasePath)"/> diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index d0147db4bd52a..e54a410fcc0a1 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -101,6 +101,7 @@ function Build { /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` /p:Publish=$publish ` + /p:__InstallAdditionalDotNetCoreFrameworks=$installAdditionalDotNetCoreFrameworks ` @properties } diff --git a/eng/common/build.sh b/eng/common/build.sh old mode 100644 new mode 100755 index 40b1e8ec73e39..792c6dbfe622a --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -200,6 +200,7 @@ function Build { /p:PerformanceTest=$performance_test \ /p:Sign=$sign \ /p:Publish=$publish \ + /p:__InstallAdditionalDotNetCoreFrameworks=$installAdditionalDotNetCoreFrameworks \ $properties ExitWithExitCode 0 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 2467ebdd4225a..81ffd16779cb1 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) { # Until we can anonymously query the BAR API for the latest arcade-services # build applied to the PROD channel, this is hardcoded. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19151.3' + $darcVersion = '1.1.0-beta.19205.4' } $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh old mode 100644 new mode 100755 index 8d63dd711b2a8..bd7eb46398640 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19151.3" +darcVersion="1.1.0-beta.19205.4" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd new file mode 100644 index 0000000000000..b1c2642e76f72 --- /dev/null +++ b/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 new file mode 100644 index 0000000000000..5987943fd6f1f --- /dev/null +++ b/eng/common/dotnet-install.ps1 @@ -0,0 +1,22 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [string] $architecture = "", + [string] $version = "Latest", + [string] $runtime = "dotnet" +) + +. $PSScriptRoot\tools.ps1 + +try { + $dotnetRoot = Join-Path $RepoRoot ".dotnet" + InstallDotNet $dotnetRoot $version $architecture $runtime $true +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh new file mode 100755 index 0000000000000..c3072c958af0d --- /dev/null +++ b/eng/common/dotnet-install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +version='Latest' +architecture='' +runtime='dotnet' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + shift +done + +. "$scriptroot/tools.sh" +dotnetRoot="$repo_root/.dotnet" +InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { + local exit_code=$? + echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index 76f57076a3232..a05b84f7987b3 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -3,7 +3,7 @@ Param( [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion = '1.1.0-beta.19169.5', # darc's version + [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version [string] $graphvizVersion = '2.38', # GraphViz version [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a81ff906f6ee..1a39a7ef3f67b 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -12,8 +12,12 @@ - - https://devdiv.pkgs.visualstudio.com/_packaging/8f470c7e-ac49-4afe-a6ee-cf784e438b93/nuget/v3/index.json; + + + https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; + + + $(RestoreSources); https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 74dd81fdc0a35..7839b70bb708e 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -43,9 +43,12 @@ parameters: # Optional: enable sending telemetry enableTelemetry: false - # Optional: define the helix repo for telemeetry (example: 'dotnet/arcade') + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') helixRepo: '' + # Optional: define the helix type for telemetry (example: 'build/product/') + helixType: '' + # Required: name of the job name: '' @@ -122,6 +125,8 @@ jobs: displayName: 'Send Helix Start Telemetry' inputs: helixRepo: ${{ parameters.helixRepo }} + ${{ if ne(parameters.helixType, '') }}: + helixType: ${{ parameters.helixType }} buildConfig: $(_BuildConfig) runAsPublic: ${{ parameters.runAsPublic }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml index 211967debab51..a0a8074282aa8 100644 --- a/eng/common/templates/phases/publish-build-assets.yml +++ b/eng/common/templates/phases/publish-build-assets.yml @@ -5,6 +5,7 @@ parameters: condition: succeeded() continueOnError: false runAsPublic: false + publishUsingPipelines: false phases: - phase: Asset_Registry_Publish displayName: Publish to Build Asset Registry @@ -36,6 +37,7 @@ phases: /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml deleted file mode 100644 index 470ab65da0c60..0000000000000 --- a/eng/common/templates/steps/helix-publish.yml +++ /dev/null @@ -1,51 +0,0 @@ -parameters: - HelixSource: 'pr/dotnet-github-anon-kaonashi-bot' - HelixType: ̓'tests/default' - HelixBuild: $(Build.BuildNumber) - HelixTargetQueues: '' - HelixAccessToken: '' - HelixPreCommands: '' - HelixPostCommands: '' - WorkItemDirectory: '' - WorkItemCommand: '' - CorrelationPayloadDirectory: '' - XUnitProjects: '' - XUnitTargetFramework: '' - XUnitRunnerVersion: '' - IncludeDotNetCli: false - DotNetCliPackageType: '' - DotNetCliVersion: '' - EnableXUnitReporter: false - WaitForWorkItemCompletion: true - condition: succeeded() - continueOnError: false - -steps: - - task: DotNetCoreCLI@2 - inputs: - command: custom - projects: eng/common/helixpublish.proj - custom: msbuild - arguments: '/bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog' - displayName: Send job to Helix - env: - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 0187597681e3e..d1ce577db5b9e 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,3 +1,4 @@ +# Please remember to update the documentation if you make changes to these parameters! parameters: HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' @@ -22,7 +23,7 @@ parameters: WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false @@ -84,4 +85,4 @@ steps: Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} \ No newline at end of file + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index de7523cae5815..f8d5754f7f244 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -119,7 +119,7 @@ function InitializeDotNetCli([bool]$install) { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" @@ -152,7 +152,7 @@ function InitializeDotNetCli([bool]$install) { } function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = "$dotnetRoot\dotnet-install.ps1" + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript @@ -162,9 +162,21 @@ function GetDotNetInstallScript([string] $dotnetRoot) { } function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { + InstallDotNet $dotnetRoot $version $architecture +} + +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot - $archArg = if ($architecture) { $architecture } else { "" } - & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg + $installParameters = @{ + Version = $version + InstallDir = $dotnetRoot + } + + if ($architecture) { $installParameters.Architecture = $architecture } + if ($runtime) { $installParameters.Runtime = $runtime } + if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + + & $installScript @installParameters if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode @@ -419,6 +431,7 @@ function InitializeToolset() { $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '' | Set-Content $proj + MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile $path = Get-Content $toolsetLocationFile -TotalCount 1 @@ -512,12 +525,24 @@ $ToolsDir = Join-Path $RepoRoot ".tools" $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json +# true if global.json contains a "runtimes" section +$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } +$installAdditionalDotNetCoreFrameworks = $globalJsonHasRuntimes + +# Don't install additional frameworks if global.json does not contain "runtimes" or building from source build +if ($null -ne $env:DotNetCoreSdkDir) { + $installAdditionalDotNetCoreFrameworks = $false +} Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir if ($ci) { + Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir" + Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir" + Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir" + $env:TEMP = $TempDir $env:TMP = $TempDir -} +} \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ecdece1f8552e..18248d04cd3a9 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -101,7 +101,7 @@ function InitializeDotNetCli { fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -115,10 +115,11 @@ function InitializeDotNetCli { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else dotnet_root="$repo_root/.dotnet" + export DOTNET_INSTALL_DIR="$dotnet_root" if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then @@ -149,16 +150,34 @@ function InitializeDotNetCli { function InstallDotNetSdk { local root=$1 local version=$2 + local architecture="" + if [[ $# == 3 ]]; then + architecture=$3 + fi + InstallDotNet "$root" "$version" $architecture +} +function InstallDotNet { + local root=$1 + local version=$2 + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local arch_arg="" - if [[ $# == 3 ]]; then - arch_arg="--architecture $3" + local archArg='' + if [[ "$#" -ge "3" ]]; then + archArg="--architecture $3" + fi + local runtimeArg='' + if [[ "$#" -ge "4" ]]; then + runtimeArg="--runtime $4" fi - bash "$install_script" --version $version --install-dir "$root" $arch_arg || { + local skipNonVersionedFilesArg="" + if [[ "$#" -ge "5" ]]; then + skipNonVersionedFilesArg="--skip-non-versioned-files" + fi + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code @@ -311,6 +330,17 @@ log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" +# determine if global.json contains a "runtimes" entry +global_json_has_runtimes=false +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +if [[ -n "$dotnetlocal_key" ]]; then + global_json_has_runtimes=true +fi +installAdditionalDotNetCoreFrameworks=$global_json_has_runtimes +# Don't install additional frameworks if global.json does not contain "runtimes" or building from source build +if [[ -n "${DotNetCoreSdkDir:-}" ]]; then + installAdditionalDotNetCoreFrameworks=false +fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then @@ -325,4 +355,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi +fi \ No newline at end of file diff --git a/global.json b/global.json index fdd7384a5dcd0..bfe125c96e149 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,15 @@ { "tools": { "dotnet": "3.0.100-preview4-010963", + "runtimes": { + "dotnet/x64": [ "$(MicrosoftNetCoreAppVersion)" ] + }, "vs": { "version": "16.0" }, "xcopy-msbuild": "16.0.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19171.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19225.4" } } From 8f31b764c202311bca7eb85b711f4c014d4673e6 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Thu, 25 Apr 2019 15:19:03 -0700 Subject: [PATCH 02/10] Update internal tool script --- eng/common/internal/Tools.csproj | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef3f67b..462d147de8f62 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -12,16 +12,12 @@ - - - https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; - - - $(RestoreSources); + + https://devdiv.pkgs.visualstudio.com/_packaging/8f470c7e-ac49-4afe-a6ee-cf784e438b93/nuget/v3/index.json; https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; - + \ No newline at end of file From 8205011fc37503fd8df3e479f5e84e36b32170c3 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Thu, 25 Apr 2019 15:48:32 -0700 Subject: [PATCH 03/10] Fix build, newer arcade sdk automatically signs corext packages --- eng/Signing.props | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index f16d58d6f4550..c2ae0c901d0a4 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -39,16 +39,4 @@ - - - - - - - - From 3d43ddbc0c52cdaaf6d973ef4b1302c5d1b21532 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Thu, 25 Apr 2019 16:19:57 -0700 Subject: [PATCH 04/10] Add back missing files --- eng/common/CheckSymbols.ps1 | 158 +++++++++++++++++++++++++ eng/common/PublishToSymbolServers.proj | 82 +++++++++++++ eng/common/SigningValidation.proj | 83 +++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 eng/common/CheckSymbols.ps1 create mode 100644 eng/common/PublishToSymbolServers.proj create mode 100644 eng/common/SigningValidation.proj diff --git a/eng/common/CheckSymbols.ps1 b/eng/common/CheckSymbols.ps1 new file mode 100644 index 0000000000000..b8d84607b89b5 --- /dev/null +++ b/eng/common/CheckSymbols.ps1 @@ -0,0 +1,158 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + throw "Input file does not exist: $PackagePath" + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + # Makes easier to reference `symbol tool` + Push-Location $SymbolToolPath + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-Error "Missing symbols for $Status modules in the package $FileName" + } + + Write-Host + } +} + +CheckSymbolsAvailable diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj new file mode 100644 index 0000000000000..5d55e312b0122 --- /dev/null +++ b/eng/common/PublishToSymbolServers.proj @@ -0,0 +1,82 @@ + + + + + + netcoreapp2.1 + + + + + + + + + + + + + + + + 3650 + true + false + + + + + + + + + + + + + + + + + diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj new file mode 100644 index 0000000000000..7045fb6fb9d42 --- /dev/null +++ b/eng/common/SigningValidation.proj @@ -0,0 +1,83 @@ + + + + + + netcoreapp2.1 + + + + + + + + $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe + + $(PackageBasePath) + signcheck.log + signcheck.errors.log + signcheck.exclusions.txt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From bb7e02717102ee809549127339a70bd1c856da7f Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Thu, 25 Apr 2019 16:56:14 -0700 Subject: [PATCH 05/10] temporary workaround https://github.com/dotnet/arcade/pull/2627/files causing failure --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index a040cd47bf298..8ee5f02e69bf0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -286,7 +286,7 @@ true true true - true + false true true - \ No newline at end of file + diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f8d5754f7f244..d86eef1e3a710 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -108,7 +108,7 @@ function InitializeDotNetCli([bool]$install) { } # Find the first path on %PATH% that contains the dotnet.exe - if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { + if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent @@ -403,6 +403,16 @@ function GetSdkTaskProject([string]$taskName) { return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" } +function InitializeNativeTools() { + if (Get-Member -InputObject $GlobalJson -Name "native-tools") { + $nativeArgs="" + if ($ci) { + $nativeArgs = "-InstallDirectory $ToolsDir" + } + Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + } +} + function InitializeToolset() { if (Test-Path variable:global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj @@ -527,12 +537,6 @@ $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json # true if global.json contains a "runtimes" section $globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } -$installAdditionalDotNetCoreFrameworks = $globalJsonHasRuntimes - -# Don't install additional frameworks if global.json does not contain "runtimes" or building from source build -if ($null -ne $env:DotNetCoreSdkDir) { - $installAdditionalDotNetCoreFrameworks = $false -} Create-Directory $ToolsetDir Create-Directory $TempDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 18248d04cd3a9..9dc565e2961ad 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -231,6 +231,17 @@ function GetNuGetPackageCachePath { _GetNuGetPackageCachePath=$NUGET_PACKAGES } +function InitializeNativeTools() { + if grep -Fq "native-tools" $global_json_file + then + local nativeArgs="" + if [[ "$ci" == true ]]; then + nativeArgs="-InstallDirectory $tools_dir" + fi + "$_script_dir/init-tools-native.sh" $nativeArgs + fi +} + function InitializeToolset { if [[ -n "${_InitializeToolset:-}" ]]; then return @@ -326,6 +337,7 @@ eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" toolset_dir="$artifacts_dir/toolset" +tools_dir="$repo_root/.tools" log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" @@ -336,11 +348,6 @@ dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true if [[ -n "$dotnetlocal_key" ]]; then global_json_has_runtimes=true fi -installAdditionalDotNetCoreFrameworks=$global_json_has_runtimes -# Don't install additional frameworks if global.json does not contain "runtimes" or building from source build -if [[ -n "${DotNetCoreSdkDir:-}" ]]; then - installAdditionalDotNetCoreFrameworks=false -fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then diff --git a/global.json b/global.json index bfe125c96e149..16d2ba38ede4f 100644 --- a/global.json +++ b/global.json @@ -10,6 +10,6 @@ "xcopy-msbuild": "16.0.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19225.4" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19253.7" } } From 06461ca39c45c4d1d6d502496f10e8d075753767 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Fri, 3 May 2019 15:43:54 -0700 Subject: [PATCH 07/10] Update x bit --- eng/common/cibuild.sh | 0 eng/common/init-tools-native.sh | 0 eng/common/msbuild.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 eng/common/cibuild.sh mode change 100644 => 100755 eng/common/init-tools-native.sh mode change 100644 => 100755 eng/common/msbuild.sh diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh old mode 100644 new mode 100755 diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh old mode 100644 new mode 100755 diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh old mode 100644 new mode 100755 From f80ecb39ced4e88309e1b83fbfc9788dc332d56b Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Fri, 3 May 2019 15:49:07 -0700 Subject: [PATCH 08/10] Temporary fix for Arcade change --- eng/targets/Settings.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/targets/Settings.props b/eng/targets/Settings.props index 2e954153b35a3..3d1bad9b033af 100644 --- a/eng/targets/Settings.props +++ b/eng/targets/Settings.props @@ -29,6 +29,8 @@ true true + true + *none* From 5594f03db18a40bf83a1dd2247935d91939bcc1b Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Fri, 3 May 2019 16:17:30 -0700 Subject: [PATCH 09/10] try to temporarily fix xlif task failure --- src/Compilers/Core/MSBuildTask/xlf/ErrorString.tr.xlf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilers/Core/MSBuildTask/xlf/ErrorString.tr.xlf b/src/Compilers/Core/MSBuildTask/xlf/ErrorString.tr.xlf index b3a57a1232880..2e5b9b3f7a53b 100644 --- a/src/Compilers/Core/MSBuildTask/xlf/ErrorString.tr.xlf +++ b/src/Compilers/Core/MSBuildTask/xlf/ErrorString.tr.xlf @@ -60,7 +60,8 @@ Failed to check the content hash of the source ref assembly '{0}': {1} {2} - Kaynak ürün kodu derleme '{0}' içerik karma denetlemek başarısız oldu: {1} (2) + Failed to check the content hash of the source ref assembly '{0}': {1} +{2} From e0841eb79687cbfcad3daa06546c19a4d8c37e86 Mon Sep 17 00:00:00 2001 From: Christopher Costa Date: Fri, 3 May 2019 16:23:44 -0700 Subject: [PATCH 10/10] disable xliff --- eng/targets/Settings.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/targets/Settings.props b/eng/targets/Settings.props index 3d1bad9b033af..7af6ccc77fd2d 100644 --- a/eng/targets/Settings.props +++ b/eng/targets/Settings.props @@ -29,7 +29,7 @@ true true - true + false *none*