From 95357f9fab2603bbe442c29a21df5e1140d2a673 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 12 Jan 2022 10:58:09 -0800 Subject: [PATCH] Revert "Streamline and simplify v3 publishing job (#8334)" This reverts commit 5de8a0ac2820510ea442174020bcaf3f59b2e3a9. --- eng/common/post-build/publish-using-darc.ps1 | 7 +- .../templates/post-build/post-build.yml | 1 + eng/promote-build.yml | 127 ++++++++-- eng/publishing/v3/common-variables.yml | 24 ++ eng/publishing/v3/nuget-validation.yml | 54 +++++ eng/publishing/v3/postbuild-checks.yml | 9 + eng/publishing/v3/publish-assets.yml | 109 +++++++++ eng/publishing/v3/publish-symbols.yml | 65 +++++ eng/publishing/v3/publish.yml | 228 ++++-------------- eng/publishing/v3/setup-maestro-vars.yml | 61 +++++ eng/publishing/v3/signing-validation.yml | 56 +++++ eng/publishing/v3/sourcelink-validation.yml | 36 +++ 12 files changed, 578 insertions(+), 199 deletions(-) create mode 100644 eng/publishing/v3/common-variables.yml create mode 100644 eng/publishing/v3/nuget-validation.yml create mode 100644 eng/publishing/v3/postbuild-checks.yml create mode 100644 eng/publishing/v3/publish-assets.yml create mode 100644 eng/publishing/v3/publish-symbols.yml create mode 100644 eng/publishing/v3/setup-maestro-vars.yml create mode 100644 eng/publishing/v3/signing-validation.yml create mode 100644 eng/publishing/v3/sourcelink-validation.yml diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index c6c38a5b45d..2427ca6b6ae 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -5,8 +5,13 @@ param( [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, + [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation, + [Parameter(Mandatory=$false)][string] $EnableSigningValidation, + [Parameter(Mandatory=$false)][string] $EnableNugetValidation, + [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, - [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters + [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters ) try { diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 8985b429c85..d1759337abe 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -263,5 +263,6 @@ stages: -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true + -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' \ No newline at end of file diff --git a/eng/promote-build.yml b/eng/promote-build.yml index 267b77d7a7a..930907f3c4a 100644 --- a/eng/promote-build.yml +++ b/eng/promote-build.yml @@ -17,20 +17,6 @@ parameters: type: string default: ' ' - - name: SymbolPublishingAdditionalParameters - displayName: Additional (MSBuild) properties for symbol publishing - type: string - default: ' ' - - - name: ArtifactsPublishingAdditionalParameters - displayName: Additional (MSBuild) properties for general asset publishing - type: string - default: ' ' - - # The parameters below here are legacy. They are passed by add-build-to-channel - # to the build pipeline, and if they are not present in the pipeline, then queueing - # will fail. Remove once add-build-to-channel has been updated to remove the parameters. - - name: EnableSourceLinkValidation displayName: Should Sourcelink validation be performed? type: boolean @@ -50,6 +36,16 @@ parameters: displayName: Should installers and checksums be published? type: boolean default: true + + - name: SymbolPublishingAdditionalParameters + displayName: Additional (MSBuild) properties for symbol publishing + type: string + default: ' ' + + - name: ArtifactsPublishingAdditionalParameters + displayName: Additional (MSBuild) properties for general asset publishing + type: string + default: ' ' - name: SigningValidationAdditionalParameters displayName: Additional (MSBuild) properties for signing validation @@ -58,10 +54,101 @@ parameters: trigger: none +variables: + _DotNetArtifactsCategory: .NETCore + +pool: + vmImage: ubuntu-latest + stages: -- template: \eng\publishing\v3\publish.yml - parameters: - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - BARBuildId: ${{ parameters.BARBuildId }} - symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }} - artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }} +- stage: prepare_promotion + displayName: Prepare for Promotion + jobs: + - job: + displayName: Validate Parameters + variables: + - template: common\templates\post-build\common-variables.yml + steps: + - checkout: none + + - task: PowerShell@2 + displayName: Validate Build & Channel + inputs: + targetType: inline + script: | + # Keeping this script inline so that we don't need to checkout the whole repo to use just one file + try { + $buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + if (!$buildInfo) { + Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!" + exit 1 + } + + $channels = ${Env:PromoteToChannelIds} -split "-" + foreach ($channelId in $channels) { + $channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)" + $channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + if (!$channelInfo) { + Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting." + exit 1 + } + } + + $azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber + $channelName = $channelInfo.name + $azureDevOpsRepository = "Unknown" + $lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/') + + if ($lastIndexOfSlash -ne -1) { + $azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1) + + # Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*' + $azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_' + } + + $buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber to channel(s) ${Env:PromoteToChannelIds}#" + + # Maximum buildnumber length is 255 chars + if ($buildNumberName.Length -GT 255) { + $buildNumberName = $buildNumberName.Substring(0, 255) + } + + Write-Host "##vso[build.updatebuildnumber]$buildNumberName" + Write-Host "##vso[build.addbuildtag]$channelName" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + +- ${{ if ge(parameters.PublishingInfraVersion, 3) }}: + - template: \eng\publishing\v3\publish.yml + parameters: + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + BARBuildId: ${{ parameters.BARBuildId }} + + enableSourceLinkValidation: ${{ parameters.EnableSourceLinkValidation }} + enableNugetValidation: ${{ parameters.EnableNugetValidation }} + enableSigningValidation: ${{ parameters.EnableSigningValidation }} + + validateDependsOn: + - prepare_promotion + + publishInstallersAndChecksums: ${{ parameters.PublishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }} + artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.SigningValidationAdditionalParameters }} diff --git a/eng/publishing/v3/common-variables.yml b/eng/publishing/v3/common-variables.yml new file mode 100644 index 00000000000..ed518e46b57 --- /dev/null +++ b/eng/publishing/v3/common-variables.yml @@ -0,0 +1,24 @@ +variables: + - group: AzureDevOps-Artifact-Feeds-Pats + - group: DotNet-Blob-Feed + - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage + - group: Publish-Build-Assets + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false \ No newline at end of file diff --git a/eng/publishing/v3/nuget-validation.yml b/eng/publishing/v3/nuget-validation.yml new file mode 100644 index 00000000000..6c9e0b8c3ef --- /dev/null +++ b/eng/publishing/v3/nuget-validation.yml @@ -0,0 +1,54 @@ +jobs: +- job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + targetType: inline + script: | + try { + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + + # `tools.ps1` requires $ci to be $true + $ci = $true + $disableConfigureToolsetImport = $true + . ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1 + + $ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\" + $PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\" + $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' + + New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force + + Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 + + & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg + } + catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ + ExitWithExitCode 1 + } diff --git a/eng/publishing/v3/postbuild-checks.yml b/eng/publishing/v3/postbuild-checks.yml new file mode 100644 index 00000000000..9382a1a40e3 --- /dev/null +++ b/eng/publishing/v3/postbuild-checks.yml @@ -0,0 +1,9 @@ +jobs: +- job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml new file mode 100644 index 00000000000..7753ebf8db3 --- /dev/null +++ b/eng/publishing/v3/publish-assets.yml @@ -0,0 +1,109 @@ +parameters: + artifactsPublishingAdditionalParameters: '' + publishInstallersAndChecksums: true + PromoteToChannelIds: '' + symbolPublishingAdditionalParameters: '' + buildQuality: 'daily' + +jobs: +- job: publish_assets + displayName: Publish Assets and Symbols + dependsOn: setupMaestroVars + timeoutInMinutes: 120 + variables: + - group: DotNet-Symbol-Server-Pats + - group: DotNetBuilds storage account tokens + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + - name: AzDOAccount + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ] + + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Assets + continueOnError: true + enabled: true + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + downloadType: 'specific' + itemPattern: | + AssetManifests/** + BlobArtifacts/MergedManifest.xml + PdbArtifacts/** + ReleaseConfigs/SymbolPublishingExclusionsFile.txt + downloadPath: '$(Build.ArtifactStagingDirectory)' + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish packages, blobs and symbols + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:PublishingInfraVersion=3 + /p:BARBuildId=$(BARBuildId) + /p:TargetChannels='${{ parameters.PromoteToChannelIds }}' + /p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + /p:NugetPath=$(NuGetExeToolPath) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key) + /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AkaMSClientId=$(akams-client-id) + /p:AkaMSClientSecret=$(akams-client-secret) + ${{ parameters.artifactsPublishingAdditionalParameters }} + /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt' + ${{ parameters.symbolPublishingAdditionalParameters}} + /p:MsdlToken=$(microsoft-symbol-server-pat) + /p:SymWebToken=$(symweb-symbol-server-pat) + /p:BuildQuality='${{ parameters.buildQuality }}' + /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' + /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' + /p:BuildId='$(AzDOBuildId)' + /p:AzureDevOpsOrg='$(AzDOAccount)' + /p:AzureProject='$(AzDOProjectName)' + /p:UseStreamingPublishing='true' + /p:StreamingPublishingMaxClients=16 + /p:NonStreamingPublishingMaxClients=12 + /p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)' + /p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)' + /p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)' + /p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)' + - template: /eng/common/templates/steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'AssetsPublishing' diff --git a/eng/publishing/v3/publish-symbols.yml b/eng/publishing/v3/publish-symbols.yml new file mode 100644 index 00000000000..10196525e95 --- /dev/null +++ b/eng/publishing/v3/publish-symbols.yml @@ -0,0 +1,65 @@ +parameters: + symbolPublishingAdditionalParameters: '' + +jobs: +- job: publish_symbols + displayName: Symbols Publishing + dependsOn: setupMaestroVars + variables: + - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Assets + continueOnError: true + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + downloadType: 'specific' + itemPattern: | + PdbArtifacts/** + BlobArtifacts/** + downloadPath: '$(Build.ArtifactStagingDirectory)' + checkDownloadedFiles: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + /p:PublishToMSDL=false + /p:PublishToSymWeb=true + ${{ parameters.symbolPublishingAdditionalParameters }} + + - template: /eng/common/templates/steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'SymbolPublishing' diff --git a/eng/publishing/v3/publish.yml b/eng/publishing/v3/publish.yml index 5fcd33637dd..ef0315cff82 100644 --- a/eng/publishing/v3/publish.yml +++ b/eng/publishing/v3/publish.yml @@ -1,189 +1,61 @@ parameters: - artifactsPublishingAdditionalParameters: '' - PromoteToChannelIds: '' BARBuildId: '' + PromoteToChannelIds: '' + + enableSourceLinkValidation: false + enableSigningValidation: true + enableNugetValidation: true + publishInstallersAndChecksums: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' - buildQuality: 'daily' + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Which stages should finish execution before post-build stages start + validateDependsOn: + - build + publishDependsOn: + - Validate stages: - - stage: publish - displayName: Publishing + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: /eng/publishing/v3/common-variables.yml jobs: - - job: publish_assets - displayName: Publish Assets and Symbols - timeoutInMinutes: 120 - variables: - - group: DotNet-Symbol-Server-Pats - - group: DotNetBuilds storage account tokens - - group: AzureDevOps-Artifact-Feeds-Pats - - group: DotNet-Blob-Feed - - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - - group: Publish-Build-Assets - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Validate and Locate Build - inputs: - targetType: inline - script: | - # Keeping this script inline so that we don't need to checkout the whole repo to use just one file - try { - $buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - if (!$buildInfo) { - Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!" - exit 1 - } - - $channels = ${Env:PromoteToChannelIds} -split "-" - $channelNames = @() - foreach ($channelId in $channels) { - $channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)" - $channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + - template: /eng/publishing/v3/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - if (!$channelInfo) { - Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting." - exit 1 - } + - template: /eng/publishing/v3/postbuild-checks.yml - $channelNames += "'$($channelInfo.name)'" - } - - $azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber - $azureDevOpsRepository = "Unknown" - $lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/') + - ${{ if eq(parameters.enableNugetValidation, 'True') }}: + - template: /eng/publishing/v3/nuget-validation.yml - if ($lastIndexOfSlash -ne -1) { - $azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1) + - ${{ if eq(parameters.enableSigningValidation, 'True') }}: + - template: /eng/publishing/v3/signing-validation.yml - # Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*' - $azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_' - } + - ${{ if eq(parameters.enableSourceLinkValidation, 'True') }}: + - template: /eng/publishing/v3/sourcelink-validation.yml - $channelNames = $channelNames -join ", " - $buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber (${Env:BARBuildId}) to channel(s) $channelNames #" - - # Maximum buildnumber length is 255 chars - if ($buildNumberName.Length -GT 255) { - $buildNumberName = $buildNumberName.Substring(0, 255) - } - - # Set tags on publishing for visibility - - Write-Host "##vso[build.updatebuildnumber]$buildNumberName" - Write-Host "##vso[build.addbuildtag]Channel(s) - $channelNames" - Write-Host "##vso[build.addbuildtag]BAR ID - ${Env:BARBuildId}" - - # Set variables used in publishing. - Write-Host "##vso[task.setvariable variable=AzDOProject]$($buildInfo.azureDevOpsProject)" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$($buildInfo.azureDevOpsBuildDefinitionId)" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$($buildInfo.azureDevOpsBuildId)" - Write-Host "##vso[task.setvariable variable=AzDOAccount]$($buildInfo.azureDevOpsAccount)" - Write-Host "##vso[task.setvariable variable=AzDOBranch]$($buildInfo.azureDevOpsBranch)" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true - enabled: true - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProject) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - AssetManifests/** - BlobArtifacts/MergedManifest.xml - PdbArtifacts/** - ReleaseConfigs/SymbolPublishingExclusionsFile.txt - downloadPath: '$(Build.ArtifactStagingDirectory)' - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - # This is necessary whenever we want to publish/restore to an AzDO private feed - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Publish packages, blobs and symbols - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:PublishingInfraVersion=3 - /p:BARBuildId=${{ parameters.BARBuildId }} - /p:TargetChannels='${{ parameters.PromoteToChannelIds }}' - /p:IsInternalBuild=${{ contains(variables['AzDOBranch'], 'internal/') }} - /p:NugetPath=$(NuGetExeToolPath) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:PublishInstallersAndChecksums=true - /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key) - /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) - /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key) - /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:AkaMSClientId=$(akams-client-id) - /p:AkaMSClientSecret=$(akams-client-secret) - ${{ parameters.artifactsPublishingAdditionalParameters }} - /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' - /p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt' - ${{ parameters.symbolPublishingAdditionalParameters}} - /p:MsdlToken=$(microsoft-symbol-server-pat) - /p:SymWebToken=$(symweb-symbol-server-pat) - /p:BuildQuality='${{ parameters.buildQuality }}' - /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)' - /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/' - /p:BuildId='$(AzDOBuildId)' - /p:AzureDevOpsOrg='$(AzDOAccount)' - /p:AzureProject='$(AzDOProject)' - /p:UseStreamingPublishing='true' - /p:StreamingPublishingMaxClients=16 - /p:NonStreamingPublishingMaxClients=12 - /p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)' - /p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)' - /p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)' - /p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)' - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'AssetsPublishing' + - stage: publish + dependsOn: ${{ parameters.publishDependsOn }} + variables: + - template: /eng/publishing/v3/common-variables.yml + displayName: Publishing + jobs: + - template: /eng/publishing/v3/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - template: /eng/publishing/v3/publish-assets.yml + parameters: + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + symbolPublishingAdditionalParameters: ${{parameters.symbolPublishingAdditionalParameters}} diff --git a/eng/publishing/v3/setup-maestro-vars.yml b/eng/publishing/v3/setup-maestro-vars.yml new file mode 100644 index 00000000000..ae2e65527b6 --- /dev/null +++ b/eng/publishing/v3/setup-maestro-vars.yml @@ -0,0 +1,61 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + variables: + - template: /eng/publishing/v3/common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - checkout: none + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + try { + dir Env: + + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + $AzureDevOpsAccount = $buildInfo.azureDevOpsAccount + + Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + Write-Host "##vso[task.setvariable variable=AzDOBuildAccount;isOutput=true]$AzureDevOpsAccount" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/publishing/v3/signing-validation.yml b/eng/publishing/v3/signing-validation.yml new file mode 100644 index 00000000000..22611ddff7f --- /dev/null +++ b/eng/publishing/v3/signing-validation.yml @@ -0,0 +1,56 @@ +jobs: +- job: + displayName: Signing Validation + dependsOn: setupMaestroVars + variables: + - template: /eng/publishing/v3/common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/templates/steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' diff --git a/eng/publishing/v3/sourcelink-validation.yml b/eng/publishing/v3/sourcelink-validation.yml new file mode 100644 index 00000000000..b290d57e424 --- /dev/null +++ b/eng/publishing/v3/sourcelink-validation.yml @@ -0,0 +1,36 @@ +jobs: +- job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + variables: + - template: /eng/publishing/v3/common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true