Revert "Streamline and simplify v3 publishing job"

eng/common/post-build/publish-using-darc.ps1

@@ -5,8 +5,13 @@ param(
   [Parameter(Mandatory=$true)][string] $MaestroToken,
   [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
+  [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
+  [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
+  [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
+  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
-  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
+  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
+  [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
 )
 
 try {

eng/common/templates/post-build/post-build.yml

@@ -263,5 +263,6 @@ stages:
             -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
             -MaestroToken '$(MaestroApiAccessToken)'
             -WaitPublishingFinish true
+            -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/promote-build.yml

@@ -17,20 +17,6 @@ parameters:
     type: string 
     default: ' '
 
-  - name: SymbolPublishingAdditionalParameters
-    displayName: Additional (MSBuild) properties for symbol publishing
-    type: string 
-    default: ' '
-
-  - name: ArtifactsPublishingAdditionalParameters
-    displayName: Additional (MSBuild) properties for general asset publishing
-    type: string 
-    default: ' '
-
-  # The parameters below here are legacy. They are passed by add-build-to-channel
-  # to the build pipeline, and if they are not present in the pipeline, then queueing
-  # will fail. Remove once add-build-to-channel has been updated to remove the parameters.
-
   - name: EnableSourceLinkValidation
     displayName: Should Sourcelink validation be performed?
     type: boolean 
@@ -50,6 +36,16 @@ parameters:
     displayName: Should installers and checksums be published?
     type: boolean 
     default: true
+
+  - name: SymbolPublishingAdditionalParameters
+    displayName: Additional (MSBuild) properties for symbol publishing
+    type: string 
+    default: ' '
+
+  - name: ArtifactsPublishingAdditionalParameters
+    displayName: Additional (MSBuild) properties for general asset publishing
+    type: string 
+    default: ' '
 
   - name: SigningValidationAdditionalParameters
     displayName: Additional (MSBuild) properties for signing validation
@@ -58,10 +54,101 @@ parameters:
 
 trigger: none
 
+variables:
+  _DotNetArtifactsCategory: .NETCore
+
+pool:
+  vmImage: ubuntu-latest
+
 stages:
-- template: \eng\publishing\v3\publish.yml
-  parameters:
-    PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-    BARBuildId: ${{ parameters.BARBuildId }}
-    symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
-    artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
+- stage: prepare_promotion
+  displayName: Prepare for Promotion
+  jobs:
+    - job:
+      displayName: Validate Parameters
+      variables:
+        - template: common\templates\post-build\common-variables.yml
+      steps:
+        - checkout: none
+
+        - task: PowerShell@2
+          displayName: Validate Build & Channel
+          inputs:
+            targetType: inline
+            script: |
+              # Keeping this script inline so that we don't need to checkout the whole repo to use just one file
+              try {
+                  $buildApiEndpoint = "$(MaestroApiEndPoint)/api/builds/${Env:BARBuildId}?api-version=$(MaestroApiVersion)"
+
+                  $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+                  $apiHeaders.Add('Accept', 'application/json')
+                  $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
+
+                  $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+
+                  if (!$buildInfo) {
+                    Write-Host "Build with BAR ID ${Env:BARBuildId} was not found in BAR!"
+                    exit 1
+                  }
+
+                  $channels = ${Env:PromoteToChannelIds} -split "-"
+                  foreach ($channelId in $channels) {
+                    $channelApiEndpoint = "$(MaestroApiEndPoint)/api/channels/${channelId}?api-version=$(MaestroApiVersion)"
+                    $channelInfo = try { Invoke-WebRequest -Method Get -Uri $channelApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+
+                    if (!$channelInfo) {
+                      Write-Host "Channel with ID ${channelId} was not found in BAR. Aborting."
+                      exit 1
+                    }
+                  }
+
+                  $azureDevOpsBuildNumber = $buildInfo.azureDevOpsBuildNumber
+                  $channelName = $channelInfo.name
+                  $azureDevOpsRepository = "Unknown"
+                  $lastIndexOfSlash = $buildInfo.azureDevOpsRepository.LastIndexOf('/')
+
+                  if ($lastIndexOfSlash -ne -1) {
+                    $azureDevOpsRepository = $buildInfo.azureDevOpsRepository.Substring($lastIndexOfSlash + 1)
+
+                    # Invalid chars in Azdo build number: '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'
+                    $azureDevOpsRepository = $azureDevOpsRepository -replace '["/:<>\\|?@*"]', '_'
+                  }
+
+                  $buildNumberName = "Promoting $azureDevOpsRepository build $azureDevOpsBuildNumber to channel(s) ${Env:PromoteToChannelIds}#"
+
+                  # Maximum buildnumber length is 255 chars
+                  if ($buildNumberName.Length -GT 255) {
+                    $buildNumberName = $buildNumberName.Substring(0, 255)
+                  }
+
+                  Write-Host "##vso[build.updatebuildnumber]$buildNumberName"
+                  Write-Host "##vso[build.addbuildtag]$channelName"
+              }
+              catch {
+                Write-Host $_
+                Write-Host $_.Exception
+                Write-Host $_.ScriptStackTrace
+                exit 1
+              }
+          env:
+            MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
+            BARBuildId: ${{ parameters.BARBuildId }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+- ${{ if ge(parameters.PublishingInfraVersion, 3) }}:
+  - template: \eng\publishing\v3\publish.yml
+    parameters:
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+      BARBuildId: ${{ parameters.BARBuildId }}
+
+      enableSourceLinkValidation: ${{ parameters.EnableSourceLinkValidation }}
+      enableNugetValidation: ${{ parameters.EnableNugetValidation }}
+      enableSigningValidation: ${{ parameters.EnableSigningValidation }}
+
+      validateDependsOn:
+        - prepare_promotion
+
+      publishInstallersAndChecksums: ${{ parameters.PublishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.SymbolPublishingAdditionalParameters }}
+      artifactsPublishingAdditionalParameters: ${{ parameters.ArtifactsPublishingAdditionalParameters }}
+      signingValidationAdditionalParameters: ${{ parameters.SigningValidationAdditionalParameters }}

eng/publishing/v3/common-variables.yml

@@ -0,0 +1,24 @@
+variables:
+  - group: AzureDevOps-Artifact-Feeds-Pats
+  - group: DotNet-Blob-Feed
+  - group: DotNet-DotNetCli-Storage
+  - group: DotNet-MSRC-Storage
+  - group: Publish-Build-Assets
+
+  # Default Maestro++ API Endpoint and API Version
+  - name: MaestroApiEndPoint
+    value: "https://maestro-prod.westus2.cloudapp.azure.com"
+  - name: MaestroApiAccessToken
+    value: $(MaestroAccessToken)
+  - name: MaestroApiVersion
+    value: "2020-02-20"
+
+  - name: SourceLinkCLIVersion
+    value: 3.0.0
+
+  # Skip component governance and codesign validation for SDL. These jobs
+  # create no content.
+  - name: skipComponentGovernanceDetection
+    value: true
+  - name: runCodesignValidationInjection
+    value: false

eng/publishing/v3/nuget-validation.yml

@@ -0,0 +1,54 @@
+jobs:
+- job:
+  displayName: NuGet Validation
+  dependsOn: setupMaestroVars
+  pool:
+    vmImage: 'windows-2019'
+  variables:
+    - name: AzDOProjectName
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+    - name: AzDOPipelineId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+    - name: AzDOBuildId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+  steps:
+    - task: DownloadBuildArtifacts@0
+      displayName: Download Package Artifacts
+      inputs:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: $(AzDOProjectName)
+        pipeline: $(AzDOPipelineId)
+        buildId: $(AzDOBuildId)
+        artifactName: PackageArtifacts
+        checkDownloadedFiles: true
+
+    - task: PowerShell@2
+      displayName: Validate
+      inputs:
+        targetType: inline
+        script: |
+          try {
+            $ErrorActionPreference = 'Stop'
+            Set-StrictMode -Version 2.0
+
+            # `tools.ps1` requires $ci to be $true
+            $ci = $true
+            $disableConfigureToolsetImport = $true
+            . ${Env:BUILD_SOURCESDIRECTORY}\eng\common\tools.ps1
+
+            $ToolDestinationPath = "${Env:AGENT_BUILDDIRECTORY}\Extract\"
+            $PackagesPath = "${Env:BUILD_ARTIFACTSTAGINGDIRECTORY}\PackageArtifacts\"
+            $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
+
+            New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
+
+            Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
+
+            & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+          } 
+          catch {
+            Write-Host $_.ScriptStackTrace
+            Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
+            ExitWithExitCode 1
+          }

eng/publishing/v3/postbuild-checks.yml

@@ -0,0 +1,9 @@
+jobs:
+- job:
+  displayName: Post-build Checks
+  dependsOn: setupMaestroVars
+  variables:
+    - name: TargetChannels
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+  pool:
+    vmImage: 'windows-2019'

eng/publishing/v3/publish-assets.yml

@@ -0,0 +1,109 @@
+parameters:
+  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: true
+  PromoteToChannelIds: ''
+  symbolPublishingAdditionalParameters: ''
+  buildQuality: 'daily'
+
+jobs:
+- job: publish_assets
+  displayName: Publish Assets and Symbols
+  dependsOn: setupMaestroVars
+  timeoutInMinutes: 120
+  variables:
+    - group: DotNet-Symbol-Server-Pats
+    - group: DotNetBuilds storage account tokens
+    - name: BARBuildId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+    - name: IsStableBuild
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+    - name: AzDOProjectName
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+    - name: AzDOPipelineId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+    - name: AzDOBuildId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    - name: AzDOAccount
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildAccount'] ]
+
+  pool:
+    vmImage: 'windows-2019'
+  steps:
+    - task: DownloadBuildArtifacts@0
+      displayName: Download Build Assets
+      continueOnError: true
+      enabled: true
+      inputs:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: $(AzDOProjectName)
+        pipeline: $(AzDOPipelineId)
+        buildId: $(AzDOBuildId)
+        downloadType: 'specific'
+        itemPattern: |
+          AssetManifests/**
+          BlobArtifacts/MergedManifest.xml
+          PdbArtifacts/**
+          ReleaseConfigs/SymbolPublishingExclusionsFile.txt
+        downloadPath: '$(Build.ArtifactStagingDirectory)'
+
+    - task: NuGetToolInstaller@1
+      displayName: 'Install NuGet.exe'
+
+    # This is necessary whenever we want to publish/restore to an AzDO private feed
+    - task: NuGetAuthenticate@0
+      displayName: 'Authenticate to AzDO Feeds'
+
+    - task: PowerShell@2
+      displayName: Enable cross-org publishing
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+        arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+    - task: PowerShell@2
+      displayName: Publish packages, blobs and symbols
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+        arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+          /p:PublishingInfraVersion=3
+          /p:BARBuildId=$(BARBuildId)
+          /p:TargetChannels='${{ parameters.PromoteToChannelIds }}'
+          /p:IsInternalBuild=${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+          /p:NugetPath=$(NuGetExeToolPath)
+          /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+          /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+          /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+          /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+          /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+          /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+          /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+          /p:InternalInstallersAzureAccountKey=$(dotnetclimsrc-access-key)
+          /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+          /p:InternalChecksumsAzureAccountKey=$(dotnetclichecksumsmsrc-storage-key)
+          /p:AzureDevOpsFeedsKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+          /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+          /p:AkaMSClientId=$(akams-client-id)
+          /p:AkaMSClientSecret=$(akams-client-secret)
+          ${{ parameters.artifactsPublishingAdditionalParameters }}
+          /p:PDBArtifactsBasePath='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+          /p:SymbolPublishingExclusionsFile='$(Build.ArtifactStagingDirectory)/ReleaseConfigs/SymbolPublishingExclusionsFile.txt'
+          ${{ parameters.symbolPublishingAdditionalParameters}}
+          /p:MsdlToken=$(microsoft-symbol-server-pat)
+          /p:SymWebToken=$(symweb-symbol-server-pat)
+          /p:BuildQuality='${{ parameters.buildQuality }}'
+          /p:AzdoApiToken='$(dn-bot-all-orgs-build-rw-code-rw)'
+          /p:ArtifactsBasePath='$(Build.ArtifactStagingDirectory)/'
+          /p:BuildId='$(AzDOBuildId)'
+          /p:AzureDevOpsOrg='$(AzDOAccount)'
+          /p:AzureProject='$(AzDOProjectName)'
+          /p:UseStreamingPublishing='true'
+          /p:StreamingPublishingMaxClients=16
+          /p:NonStreamingPublishingMaxClients=12
+          /p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)'
+          /p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)'
+          /p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)'
+          /p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)'
+    - template: /eng/common/templates/steps/publish-logs.yml
+      parameters:
+        StageLabel: '${{ parameters.stageName }}'
+        JobLabel: 'AssetsPublishing'