From cb63eaf5b92d87e569293305c4266e48a11be87a Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 2 Feb 2022 07:19:35 -0800 Subject: [PATCH] [release/6.0] Port 1ES pool changes to 6.0 (#8408) * Revert "Revert "Streamline and simplify v3 publishing job"" (#8340) * Revert "Revert "Streamline and simplify v3 publishing job (#8334)" (#8339)" This reverts commit 7189b4a95b539a0fc84324a2d8cda2806eb4ec1f. * Remove parameter usage * Use 1ES pools in templates that run internally (#8344) * Use 1ES pools in templates that run internally - Use the 1ES pools in our templates, jobs, etc. - To reduce some of the cost of this, remove the "Setup Maestro Vars" job and replace with a step so that we don't end up with an additional allocation in those cases. - Call setup maestro vars to populate the azdo params in execute-sdl * Fix DevDiv promotion job (#8370) - Ensure devdiv promotion job uses devdiv pools - Use powershell core (the pools don't have IE initialized so we have to use basic parsing, which is default on powershell core) * Fixup common templates to be compatible with devdiv pools (#8371) In an earlier change, hosted pool usage was moved. This didn't account for differences in pools in devdiv and dnceng. * Fix oneloc template (#8381) The OneLoc template isn't used in the official build when not on main, so validation got skipped. --- azure-pipelines-code-mirror.yml | 3 +- azure-pipelines-codeql.yml | 3 +- azure-pipelines-merge-mirror.yml | 3 +- azure-pipelines-weekly.yaml | 4 +- eng/common/post-build/publish-using-darc.ps1 | 28 +-- eng/common/templates/job/execute-sdl.yml | 23 +- eng/common/templates/job/onelocbuild.yml | 18 +- eng/common/templates/jobs/jobs.yml | 10 +- .../templates/post-build/common-variables.yml | 69 ----- .../templates/post-build/post-build.yml | 99 ++++---- .../post-build/setup-maestro-vars.yml | 124 +++++---- eng/promote-build.yml | 124 ++------- eng/publishing/v3/common-variables.yml | 24 -- eng/publishing/v3/nuget-validation.yml | 54 ---- eng/publishing/v3/postbuild-checks.yml | 9 - eng/publishing/v3/publish-assets.yml | 104 -------- eng/publishing/v3/publish-symbols.yml | 65 ----- eng/publishing/v3/publish.yml | 237 ++++++++++++++---- eng/publishing/v3/setup-maestro-vars.yml | 61 ----- eng/publishing/v3/signing-validation.yml | 56 ----- eng/publishing/v3/sourcelink-validation.yml | 36 --- 21 files changed, 361 insertions(+), 793 deletions(-) delete mode 100644 eng/publishing/v3/common-variables.yml delete mode 100644 eng/publishing/v3/nuget-validation.yml delete mode 100644 eng/publishing/v3/postbuild-checks.yml delete mode 100644 eng/publishing/v3/publish-assets.yml delete mode 100644 eng/publishing/v3/publish-symbols.yml delete mode 100644 eng/publishing/v3/setup-maestro-vars.yml delete mode 100644 eng/publishing/v3/signing-validation.yml delete mode 100644 eng/publishing/v3/sourcelink-validation.yml diff --git a/azure-pipelines-code-mirror.yml b/azure-pipelines-code-mirror.yml index f08bb58770d..ed6f484a96d 100644 --- a/azure-pipelines-code-mirror.yml +++ b/azure-pipelines-code-mirror.yml @@ -10,7 +10,8 @@ jobs: jobs: - job: Merge_GitHub_to_Azure_DevOps pool: - vmImage: windows-2019 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 variables: - name: WorkingDirectoryName value: repo-dir diff --git a/azure-pipelines-codeql.yml b/azure-pipelines-codeql.yml index 56ce5e92713..c5b95ea3dab 100644 --- a/azure-pipelines-codeql.yml +++ b/azure-pipelines-codeql.yml @@ -23,7 +23,8 @@ stages: - job: Windows_NT_CSharp timeoutInMinutes: 90 pool: - vmImage: windows-2019 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - checkout: self diff --git a/azure-pipelines-merge-mirror.yml b/azure-pipelines-merge-mirror.yml index c820598fb72..1a31abe05bd 100644 --- a/azure-pipelines-merge-mirror.yml +++ b/azure-pipelines-merge-mirror.yml @@ -10,7 +10,8 @@ jobs: jobs: - job: Merge_GitHub_to_Azure_DevOps pool: - vmImage: windows-2019 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 variables: - name: WorkingDirectoryName value: repo-dir diff --git a/azure-pipelines-weekly.yaml b/azure-pipelines-weekly.yaml index d1ce3994098..6feba661c27 100644 --- a/azure-pipelines-weekly.yaml +++ b/azure-pipelines-weekly.yaml @@ -14,7 +14,9 @@ stages: jobs: - job: Synchronize pool: - vmImage: windows-2019 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: - task: UseDotNet@2 displayName: Install Correct .NET Version diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 2427ca6b6ae..8508397d776 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -5,13 +5,8 @@ param( [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, - [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation, - [Parameter(Mandatory=$false)][string] $EnableSigningValidation, - [Parameter(Mandatory=$false)][string] $EnableNugetValidation, - [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, - [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, - [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters + [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters ) try { @@ -35,27 +30,6 @@ try { $optionalParams.Add("--no-wait") | Out-Null } - if ("false" -ne $PublishInstallersAndChecksums) { - $optionalParams.Add("--publish-installers-and-checksums") | Out-Null - } - - if ("true" -eq $EnableNugetValidation) { - $optionalParams.Add("--validate-nuget") | Out-Null - } - - if ("true" -eq $EnableSourceLinkValidation) { - $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null - } - - if ("true" -eq $EnableSigningValidation) { - $optionalParams.Add("--validate-signingchecksums") | Out-Null - - if ("" -ne $SigningValidationAdditionalParameters) { - $optionalParams.Add("--signing-validation-parameters") | Out-Null - $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null - } - } - & $darc add-build-to-channel ` --id $buildId ` --publishing-infra-version $PublishingInfraVersion ` diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 0ca2afe6961..d0a1ea8b0f2 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -29,14 +29,6 @@ parameters: # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, # not pipeline artifacts, so doesn't affect the use of this parameter. pipelineArtifactNames: [] - # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be - # downloaded from. By default, uses runtime expressions to decide based on the variables set by - # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are - # running without Maestro++/BAR involved, or to download artifacts from a specific existing build - # to iterate quickly on SDL changes. - AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] jobs: - job: Run_SDL @@ -60,15 +52,20 @@ jobs: - name: GuardianPackagesConfigFile value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: - # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019. - ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}: - vmImage: windows-2019 - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - vmImage: windows-2019 + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - checkout: self clean: true + - template: /eng/common/templates/post-build/setup-maestro-vars.yml + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: - ${{ each artifactName in parameters.artifactNames }}: diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 069098b0a06..9d1e3042d8a 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -3,9 +3,8 @@ parameters: dependsOn: '' # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: - vmImage: windows-2019 - + pool: '' + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) @@ -31,7 +30,18 @@ jobs: displayName: OneLocBuild - pool: ${{ parameters.pool }} + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 variables: - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 4dfa68f92b5..70d44735ace 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -87,7 +87,15 @@ jobs: - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete pool: - vmImage: windows-2019 + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 956ed7eb904..1ac7f49a43c 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -4,54 +4,6 @@ variables: - group: DotNet-DotNetCli-Storage - group: DotNet-MSRC-Storage - group: Publish-Build-Assets - - # .NET Core 3.1 Dev - - name: PublicDevRelease_31_Channel_Id - value: 128 - - # .NET 5 Dev - - name: Net_5_Dev_Channel_Id - value: 131 - - # .NET Eng - Validation - - name: Net_Eng_Validation_Channel_Id - value: 9 - - # .NET Eng - Latest - - name: Net_Eng_Latest_Channel_Id - value: 2 - - # .NET 3 Eng - Validation - - name: NET_3_Eng_Validation_Channel_Id - value: 390 - - # .NET 3 Eng - - name: NetCore_3_Tools_Channel_Id - value: 344 - - # .NET Core 3.0 Internal Servicing - - name: InternalServicing_30_Channel_Id - value: 184 - - # .NET Core 3.0 Release - - name: PublicRelease_30_Channel_Id - value: 19 - - # .NET Core 3.1 Release - - name: PublicRelease_31_Channel_Id - value: 129 - - # General Testing - - name: GeneralTesting_Channel_Id - value: 529 - - # .NET Core 3.1 Blazor Features - - name: NetCore_31_Blazor_Features_Channel_Id - value: 531 - - # .NET Core Experimental - - name: NetCore_Experimental_Channel_Id - value: 562 # Whether the build is internal or not - name: IsInternalBuild @@ -70,26 +22,5 @@ variables: - name: SymbolToolVersion value: 1.0.1 - # Feed Configurations - # These should include the suffix "/index.json" - - # Default locations for Installers and checksums - # Public Locations - - name: ChecksumsBlobFeedUrl - value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json - - name: InstallersBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - - # Private Locations - - name: InternalChecksumsBlobFeedUrl - value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json - - name: InternalChecksumsBlobFeedKey - value: $(dotnetclichecksumsmsrc-storage-key) - - - name: InternalInstallersBlobFeedUrl - value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json - - name: InternalInstallersBlobFeedKey - value: $(dotnetclimsrc-access-key) - - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index d1759337abe..24e92a275a4 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -90,25 +90,25 @@ stages: variables: - template: common-variables.yml jobs: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: displayName: NuGet Validation - dependsOn: setupMaestroVars condition: eq( ${{ parameters.enableNugetValidation }}, 'true') pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -129,19 +129,22 @@ stages: - job: displayName: Signing Validation - dependsOn: setupMaestroVars condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: - vmImage: 'windows-2019' + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -186,19 +189,22 @@ stages: - job: displayName: SourceLink Validation - dependsOn: setupMaestroVars condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: - vmImage: 'windows-2019' + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: DownloadBuildArtifacts@0 displayName: Download Blob Artifacts inputs: @@ -224,7 +230,6 @@ stages: - template: /eng/common/templates/job/execute-sdl.yml parameters: enable: ${{ parameters.SDLValidationParameters.enable }} - dependsOn: setupMaestroVars additionalParameters: ${{ parameters.SDLValidationParameters.params }} continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} @@ -239,21 +244,24 @@ stages: variables: - template: common-variables.yml jobs: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: displayName: Publish Using Darc - dependsOn: setupMaestroVars timeoutInMinutes: 120 - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] pool: - vmImage: 'windows-2019' + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - task: PowerShell@2 displayName: Publish Using Darc inputs: @@ -263,6 +271,5 @@ stages: -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true - -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 4a22b2e6f6d..0c87f149a4a 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -2,77 +2,69 @@ parameters: BARBuildId: '' PromoteToChannelIds: '' -jobs: -- job: setupMaestroVars - displayName: Setup Maestro Vars - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - checkout: none - - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs inputs: - targetType: inline - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" - Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/promote-build.yml b/eng/promote-build.yml index 4bcb06845f4..267b77d7a7a 100644 --- a/eng/promote-build.yml +++ b/eng/promote-build.yml @@ -17,6 +17,20 @@ parameters: type: string default: ' ' + - name: SymbolPublishingAdditionalParameters + displayName: Additional (MSBuild) properties for symbol publishing + type: string + default: ' ' + + - name: ArtifactsPublishingAdditionalParameters + displayName: Additional (MSBuild) properties for general asset publishing + type: string + default: ' ' + + # The parameters below here are legacy. They are passed by add-build-to-channel + # to the build pipeline, and if they are not present in the pipeline, then queueing + # will fail. Remove once add-build-to-channel has been updated to remove the parameters. + - name: EnableSourceLinkValidation displayName: Should Sourcelink validation be performed? type: boolean @@ -36,16 +50,6 @@ parameters: displayName: Should installers and checksums be published? type: boolean default: true - - - name: SymbolPublishingAdditionalParameters - displayName: Additional (MSBuild) properties for symbol publishing - type: string - default: ' ' - - - name: ArtifactsPublishingAdditionalParameters - displayName: Additional (MSBuild) properties for general asset publishing - type: string - default: ' ' - name: SigningValidationAdditionalParameters displayName: Additional (MSBuild) properties for signing validation @@ -54,98 +58,10 @@ parameters: trigger: none -variables: - _DotNetArtifactsCategory: .NETCore - stages: -- stage: prepare_promotion - displayName: Prepare for Promotion - jobs: - - job: - displayName: Validate Parameters - variables: - - template: common\templates\post-build\common-variables.yml - steps: - - checkout: none - - - task: PowerShell@2 - displayName: Validate Build & Channel - inputs: - targetType: inline - script: | - # Keeping this script inline so that we don't need to checkout the whole repo to use just one file - try { - $buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - if (!$buildInfo) { - Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!" - exit 1 - } - - $channels = ${Env:PromoteToChannelIds} -split "-" - foreach ($channelId in $channels) { - $channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)" - $channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - if (!$channelInfo) { - Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting." - exit 1 - } - } - - $azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber - $channelName = $channelInfo.name - $azureDevOpsRepository = "Unknown" - $lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/') - - if ($lastIndexOfSlash -ne -1) { - $azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1) - - # Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*' - $azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_' - } - - $buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber to channel(s) ${Env:PromoteToChannelIds}#" - - # Maximum buildnumber length is 255 chars - if ($buildNumberName.Length -GT 255) { - $buildNumberName = $buildNumberName.Substring(0, 255) - } - - Write-Host "##vso[build.updatebuildnumber]$buildNumberName" - Write-Host "##vso[build.addbuildtag]$channelName" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - -- ${{ if ge(parameters.PublishingInfraVersion, 3) }}: - - template: \eng\publishing\v3\publish.yml - parameters: - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - BARBuildId: ${{ parameters.BARBuildId }} - - enableSourceLinkValidation: ${{ parameters.EnableSourceLinkValidation }} - enableNugetValidation: ${{ parameters.EnableNugetValidation }} - enableSigningValidation: ${{ parameters.EnableSigningValidation }} - - validateDependsOn: - - prepare_promotion - - publishInstallersAndChecksums: ${{ parameters.PublishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }} - artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.SigningValidationAdditionalParameters }} +- template: \eng\publishing\v3\publish.yml + parameters: + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + BARBuildId: ${{ parameters.BARBuildId }} + symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }} diff --git a/eng/publishing/v3/common-variables.yml b/eng/publishing/v3/common-variables.yml deleted file mode 100644 index ed518e46b57..00000000000 --- a/eng/publishing/v3/common-variables.yml +++ /dev/null @@ -1,24 +0,0 @@ -variables: - - group: AzureDevOps-Artifact-Feeds-Pats - - group: DotNet-Blob-Feed - - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - - group: Publish-Build-Assets - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - # Skip component governance and codesign validation for SDL. These jobs - # create no content. - - name: skipComponentGovernanceDetection - value: true - - name: runCodesignValidationInjection - value: false \ No newline at end of file diff --git a/eng/publishing/v3/nuget-validation.yml b/eng/publishing/v3/nuget-validation.yml deleted file mode 100644 index 6c9e0b8c3ef..00000000000 --- a/eng/publishing/v3/nuget-validation.yml +++ /dev/null @@ -1,54 +0,0 @@ -jobs: -- job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - targetType: inline - script: | - try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - - # `tools.ps1` requires $ci to be $true - $ci = $true - $disableConfigureToolsetImport = $true - . ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1 - - $ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\" - $PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\" - $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' - - New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force - - Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 - - & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg - } - catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ - ExitWithExitCode 1 - } diff --git a/eng/publishing/v3/postbuild-checks.yml b/eng/publishing/v3/postbuild-checks.yml deleted file mode 100644 index 9382a1a40e3..00000000000 --- a/eng/publishing/v3/postbuild-checks.yml +++ /dev/null @@ -1,9 +0,0 @@ -jobs: -- job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml deleted file mode 100644 index 60dca2adcde..00000000000 --- a/eng/publishing/v3/publish-assets.yml +++ /dev/null @@ -1,104 +0,0 @@ -parameters: - artifactsPublishingAdditionalParameters: '' - publishInstallersAndChecksums: true - PromoteToChannelIds: '' - symbolPublishingAdditionalParameters: '' - buildQuality: 'daily' - -jobs: -- job: publish_assets - displayName: Publish Assets and Symbols - dependsOn: setupMaestroVars - timeoutInMinutes: 120 - variables: - - group: DotNet-Symbol-Server-Pats - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - name: IsStableBuild - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - - name: AzDOAccount - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ] - - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true - enabled: true - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - AssetManifests/** - BlobArtifacts/MergedManifest.xml - PdbArtifacts/** - ReleaseConfigs/SymbolPublishingExclusionsFile.txt - downloadPath: '$(Build.ArtifactStagingDirectory)' - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - # This is necessary whenever we want to publish/restore to an AzDO private feed - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Publish packages, blobs and symbols - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:PublishingInfraVersion=3 - /p:BARBuildId=$(BARBuildId) - /p:TargetChannels='${{ parameters.PromoteToChannelIds }}' - /p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - /p:NugetPath=$(NuGetExeToolPath) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} - /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key) - /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) - /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key) - /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:AkaMSClientId=$(akams-client-id) - /p:AkaMSClientSecret=$(akams-client-secret) - ${{ parameters.artifactsPublishingAdditionalParameters }} - /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' - /p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt' - ${{ parameters.symbolPublishingAdditionalParameters}} - /p:MsdlToken=$(microsoft-symbol-server-pat) - /p:SymWebToken=$(symweb-symbol-server-pat) - /p:BuildQuality='${{ parameters.buildQuality }}' - /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' - /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' - /p:BuildId='$(AzDOBuildId)' - /p:AzureDevOpsOrg='$(AzDOAccount)' - /p:AzureProject='$(AzDOProjectName)' - /p:UseStreamingPublishing='true' - /p:StreamingPublishingMaxClients=16 - /p:NonStreamingPublishingMaxClients=12 - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'AssetsPublishing' diff --git a/eng/publishing/v3/publish-symbols.yml b/eng/publishing/v3/publish-symbols.yml deleted file mode 100644 index 10196525e95..00000000000 --- a/eng/publishing/v3/publish-symbols.yml +++ /dev/null @@ -1,65 +0,0 @@ -parameters: - symbolPublishingAdditionalParameters: '' - -jobs: -- job: publish_symbols - displayName: Symbols Publishing - dependsOn: setupMaestroVars - variables: - - group: DotNet-Symbol-Server-Pats - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PdbArtifacts/** - BlobArtifacts/** - downloadPath: '$(Build.ArtifactStagingDirectory)' - checkDownloadedFiles: true - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - /p:Configuration=Release - /p:PublishToMSDL=false - /p:PublishToSymWeb=true - ${{ parameters.symbolPublishingAdditionalParameters }} - - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'SymbolPublishing' diff --git a/eng/publishing/v3/publish.yml b/eng/publishing/v3/publish.yml index ef0315cff82..783701729d0 100644 --- a/eng/publishing/v3/publish.yml +++ b/eng/publishing/v3/publish.yml @@ -1,61 +1,198 @@ parameters: - BARBuildId: '' + artifactsPublishingAdditionalParameters: '' PromoteToChannelIds: '' - - enableSourceLinkValidation: false - enableSigningValidation: true - enableNugetValidation: true - publishInstallersAndChecksums: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation + BARBuildId: '' symbolPublishingAdditionalParameters: '' - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Which stages should finish execution before post-build stages start - validateDependsOn: - - build - publishDependsOn: - - Validate + buildQuality: 'daily' stages: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: /eng/publishing/v3/common-variables.yml + - stage: publish + displayName: Publishing jobs: - - template: /eng/publishing/v3/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + - job: publish_assets + displayName: Publish Assets and Symbols + timeoutInMinutes: 120 + variables: + - group: DotNet-Symbol-Server-Pats + - group: DotNetBuilds storage account tokens + - group: AzureDevOps-Artifact-Feeds-Pats + - group: DotNet-Blob-Feed + - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage + - group: Publish-Build-Assets + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" - - template: /eng/publishing/v3/postbuild-checks.yml + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 - - ${{ if eq(parameters.enableNugetValidation, 'True') }}: - - template: /eng/publishing/v3/nuget-validation.yml + steps: + - task: PowerShell@2 + displayName: Validate and Locate Build + inputs: + targetType: inline + pwsh: true + script: | + # Keeping this script inline so that we don't need to checkout the whole repo to use just one file + try { + $buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)" - - ${{ if eq(parameters.enableSigningValidation, 'True') }}: - - template: /eng/publishing/v3/signing-validation.yml + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - ${{ if eq(parameters.enableSourceLinkValidation, 'True') }}: - - template: /eng/publishing/v3/sourcelink-validation.yml + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - stage: publish - dependsOn: ${{ parameters.publishDependsOn }} - variables: - - template: /eng/publishing/v3/common-variables.yml - displayName: Publishing - jobs: - - template: /eng/publishing/v3/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - template: /eng/publishing/v3/publish-assets.yml - parameters: - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - symbolPublishingAdditionalParameters: ${{parameters.symbolPublishingAdditionalParameters}} + if (!$buildInfo) { + Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!" + exit 1 + } + + $channels = ${Env:PromoteToChannelIds} -split "-" + $channelNames = @() + foreach ($channelId in $channels) { + $channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)" + $channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + if (!$channelInfo) { + Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting." + exit 1 + } + + $channelNames += "'$($channelInfo.name)'" + } + + $azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber + $azureDevOpsRepository = "Unknown" + $lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/') + + if ($lastIndexOfSlash -ne -1) { + $azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1) + + # Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*' + $azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_' + } + + $channelNames = $channelNames -join ", " + $buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber (${Env:BARBuildId}) to channel(s) $channelNames #" + + # Maximum buildnumber length is 255 chars + if ($buildNumberName.Length -GT 255) { + $buildNumberName = $buildNumberName.Substring(0, 255) + } + + # Set tags on publishing for visibility + + Write-Host "##vso[build.updatebuildnumber]$buildNumberName" + Write-Host "##vso[build.addbuildtag]Channel(s) - $channelNames" + Write-Host "##vso[build.addbuildtag]BAR ID - ${Env:BARBuildId}" + + # Set variables used in publishing. + Write-Host "##vso[task.setvariable variable=AzDOProject]$($buildInfo.azureDevOpsProject)" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$($buildInfo.azureDevOpsBuildDefinitionId)" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$($buildInfo.azureDevOpsBuildId)" + Write-Host "##vso[task.setvariable variable=AzDOAccount]$($buildInfo.azureDevOpsAccount)" + Write-Host "##vso[task.setvariable variable=AzDOBranch]$($buildInfo.azureDevOpsBranch)" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Build Assets + continueOnError: true + enabled: true + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProject) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + downloadType: 'specific' + itemPattern: | + AssetManifests/** + BlobArtifacts/MergedManifest.xml + PdbArtifacts/** + ReleaseConfigs/SymbolPublishingExclusionsFile.txt + downloadPath: '$(Build.ArtifactStagingDirectory)' + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish packages, blobs and symbols + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:PublishingInfraVersion=3 + /p:BARBuildId=${{ parameters.BARBuildId }} + /p:TargetChannels='${{ parameters.PromoteToChannelIds }}' + /p:IsInternalBuild=${{ contains(variables['AzDOBranch'], 'internal/') }} + /p:NugetPath=$(NuGetExeToolPath) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:PublishInstallersAndChecksums=true + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key) + /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AkaMSClientId=$(akams-client-id) + /p:AkaMSClientSecret=$(akams-client-secret) + ${{ parameters.artifactsPublishingAdditionalParameters }} + /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt' + ${{ parameters.symbolPublishingAdditionalParameters}} + /p:MsdlToken=$(microsoft-symbol-server-pat) + /p:SymWebToken=$(symweb-symbol-server-pat) + /p:BuildQuality='${{ parameters.buildQuality }}' + /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' + /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' + /p:BuildId='$(AzDOBuildId)' + /p:AzureDevOpsOrg='$(AzDOAccount)' + /p:AzureProject='$(AzDOProject)' + /p:UseStreamingPublishing='true' + /p:StreamingPublishingMaxClients=16 + /p:NonStreamingPublishingMaxClients=12 + /p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)' + /p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)' + /p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)' + /p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)' + - template: /eng/common/templates/steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'AssetsPublishing' diff --git a/eng/publishing/v3/setup-maestro-vars.yml b/eng/publishing/v3/setup-maestro-vars.yml deleted file mode 100644 index ae2e65527b6..00000000000 --- a/eng/publishing/v3/setup-maestro-vars.yml +++ /dev/null @@ -1,61 +0,0 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - -jobs: -- job: setupMaestroVars - displayName: Setup Maestro Vars - variables: - - template: /eng/publishing/v3/common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - checkout: none - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - script: | - try { - dir Env: - - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount - - Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" - Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/publishing/v3/signing-validation.yml b/eng/publishing/v3/signing-validation.yml deleted file mode 100644 index 22611ddff7f..00000000000 --- a/eng/publishing/v3/signing-validation.yml +++ /dev/null @@ -1,56 +0,0 @@ -jobs: -- job: - displayName: Signing Validation - dependsOn: setupMaestroVars - variables: - - template: /eng/publishing/v3/common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' diff --git a/eng/publishing/v3/sourcelink-validation.yml b/eng/publishing/v3/sourcelink-validation.yml deleted file mode 100644 index b290d57e424..00000000000 --- a/eng/publishing/v3/sourcelink-validation.yml +++ /dev/null @@ -1,36 +0,0 @@ -jobs: -- job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - variables: - - template: /eng/publishing/v3/common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true