Skip to content

Commit

Permalink
Revert "Streamline and simplify v3 publishing job (#8334)"
Browse files Browse the repository at this point in the history
This reverts commit 5de8a0a.
  • Loading branch information
mmitche authored Jan 12, 2022
1 parent 5de8a0a commit 95357f9
Show file tree
Hide file tree
Showing 12 changed files with 578 additions and 199 deletions.
7 changes: 6 additions & 1 deletion eng/common/post-build/publish-using-darc.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,13 @@ param(
[Parameter(Mandatory=$true)][string] $MaestroToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
[Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
[Parameter(Mandatory=$false)][string] $EnableSigningValidation,
[Parameter(Mandatory=$false)][string] $EnableNugetValidation,
[Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
)

try {
Expand Down
1 change: 1 addition & 0 deletions eng/common/templates/post-build/post-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -263,5 +263,6 @@ stages:
-AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish true
-PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
127 changes: 107 additions & 20 deletions eng/promote-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,6 @@ parameters:
type: string
default: ' '

- name: SymbolPublishingAdditionalParameters
displayName: Additional (MSBuild) properties for symbol publishing
type: string
default: ' '

- name: ArtifactsPublishingAdditionalParameters
displayName: Additional (MSBuild) properties for general asset publishing
type: string
default: ' '

# The parameters below here are legacy. They are passed by add-build-to-channel
# to the build pipeline, and if they are not present in the pipeline, then queueing
# will fail. Remove once add-build-to-channel has been updated to remove the parameters.

- name: EnableSourceLinkValidation
displayName: Should Sourcelink validation be performed?
type: boolean
Expand All @@ -50,6 +36,16 @@ parameters:
displayName: Should installers and checksums be published?
type: boolean
default: true

- name: SymbolPublishingAdditionalParameters
displayName: Additional (MSBuild) properties for symbol publishing
type: string
default: ' '

- name: ArtifactsPublishingAdditionalParameters
displayName: Additional (MSBuild) properties for general asset publishing
type: string
default: ' '

- name: SigningValidationAdditionalParameters
displayName: Additional (MSBuild) properties for signing validation
Expand All @@ -58,10 +54,101 @@ parameters:

trigger: none

variables:
_DotNetArtifactsCategory: .NETCore

pool:
vmImage: ubuntu-latest

stages:
- template: \eng\publishing\v3\publish.yml
parameters:
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
BARBuildId: ${{ parameters.BARBuildId }}
symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
- stage: prepare_promotion
displayName: Prepare for Promotion
jobs:
- job:
displayName: Validate Parameters
variables:
- template: common\templates\post-build\common-variables.yml
steps:
- checkout: none

- task: PowerShell@2
displayName: Validate Build & Channel
inputs:
targetType: inline
script: |
# Keeping this script inline so that we don't need to checkout the whole repo to use just one file
try {
$buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)"
$apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
$apiHeaders.Add('Accept', 'application/json')
$apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
$buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
if (!$buildInfo) {
Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!"
exit 1
}
$channels = ${Env:PromoteToChannelIds} -split "-"
foreach ($channelId in $channels) {
$channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)"
$channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
if (!$channelInfo) {
Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting."
exit 1
}
}
$azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber
$channelName = $channelInfo.name
$azureDevOpsRepository = "Unknown"
$lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/')
if ($lastIndexOfSlash -ne -1) {
$azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1)
# Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'
$azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_'
}
$buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber to channel(s) ${Env:PromoteToChannelIds}#"
# Maximum buildnumber length is 255 chars
if ($buildNumberName.Length -GT 255) {
$buildNumberName = $buildNumberName.Substring(0, 255)
}
Write-Host "##vso[build.updatebuildnumber]$buildNumberName"
Write-Host "##vso[build.addbuildtag]$channelName"
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
exit 1
}
env:
MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}

- ${{ if ge(parameters.PublishingInfraVersion, 3) }}:
- template: \eng\publishing\v3\publish.yml
parameters:
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
BARBuildId: ${{ parameters.BARBuildId }}

enableSourceLinkValidation: ${{ parameters.EnableSourceLinkValidation }}
enableNugetValidation: ${{ parameters.EnableNugetValidation }}
enableSigningValidation: ${{ parameters.EnableSigningValidation }}

validateDependsOn:
- prepare_promotion

publishInstallersAndChecksums: ${{ parameters.PublishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
signingValidationAdditionalParameters: ${{ parameters.SigningValidationAdditionalParameters }}
24 changes: 24 additions & 0 deletions eng/publishing/v3/common-variables.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage
- group: Publish-Build-Assets

# Default Maestro++ API Endpoint and API Version
- name: MaestroApiEndPoint
value: "https://maestro-prod.westus2.cloudapp.azure.com"
- name: MaestroApiAccessToken
value: $(MaestroAccessToken)
- name: MaestroApiVersion
value: "2020-02-20"

- name: SourceLinkCLIVersion
value: 3.0.0

# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
value: true
- name: runCodesignValidationInjection
value: false
54 changes: 54 additions & 0 deletions eng/publishing/v3/nuget-validation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
jobs:
- job:
displayName: NuGet Validation
dependsOn: setupMaestroVars
pool:
vmImage: 'windows-2019'
variables:
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true

- task: PowerShell@2
displayName: Validate
inputs:
targetType: inline
script: |
try {
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
# `tools.ps1` requires $ci to be $true
$ci = $true
$disableConfigureToolsetImport = $true
. ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1
$ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\"
$PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\"
$url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
& ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
ExitWithExitCode 1
}
9 changes: 9 additions & 0 deletions eng/publishing/v3/postbuild-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
jobs:
- job:
displayName: Post-build Checks
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool:
vmImage: 'windows-2019'
109 changes: 109 additions & 0 deletions eng/publishing/v3/publish-assets.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
parameters:
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: true
PromoteToChannelIds: ''
symbolPublishingAdditionalParameters: ''
buildQuality: 'daily'

jobs:
- job: publish_assets
displayName: Publish Assets and Symbols
dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables:
- group: DotNet-Symbol-Server-Pats
- group: DotNetBuilds storage account tokens
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- name: AzDOAccount
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]

pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
enabled: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
AssetManifests/**
BlobArtifacts/MergedManifest.xml
PdbArtifacts/**
ReleaseConfigs/SymbolPublishingExclusionsFile.txt
downloadPath: '$(Build.ArtifactStagingDirectory)'

- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'

# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'

- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)

- task: PowerShell@2
displayName: Publish packages, blobs and symbols
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:PublishingInfraVersion=3
/p:BARBuildId=$(BARBuildId)
/p:TargetChannels='${{ parameters.PromoteToChannelIds }}'
/p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
/p:NugetPath=$(NuGetExeToolPath)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key)
/p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:AkaMSClientId=$(akams-client-id)
/p:AkaMSClientSecret=$(akams-client-secret)
${{ parameters.artifactsPublishingAdditionalParameters }}
/p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt'
${{ parameters.symbolPublishingAdditionalParameters}}
/p:MsdlToken=$(microsoft-symbol-server-pat)
/p:SymWebToken=$(symweb-symbol-server-pat)
/p:BuildQuality='${{ parameters.buildQuality }}'
/p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)'
/p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/'
/p:BuildId='$(AzDOBuildId)'
/p:AzureDevOpsOrg='$(AzDOAccount)'
/p:AzureProject='$(AzDOProjectName)'
/p:UseStreamingPublishing='true'
/p:StreamingPublishingMaxClients=16
/p:NonStreamingPublishingMaxClients=12
/p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)'
/p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)'
/p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)'
/p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)'
- template: /eng/common/templates/steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'AssetsPublishing'
Loading

0 comments on commit 95357f9

Please sign in to comment.