Switch to Arcade yaml stage-based publishing with custom steps

azure-pipelines.yml

@@ -8,7 +8,6 @@ trigger:
 pr:
 - master
 - release/3.0
-- dev/arcade-migration
 
 name: $(Date:yyyyMMdd)$(Rev:.r)
 
@@ -22,6 +21,12 @@ variables:
   - name: OfficialBuildId
     value: $(Build.BuildNumber)
 
+  # Set the target blob feed for package publish during official and validation builds.
+  - name: _DotNetArtifactsCategory
+    value: .NETCore
+  - name: _DotNetValidationArtifactsCategory
+    value: .NETCoreValidation
+
   # Produce test-signed build for PR and Public builds
   - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
     - name: SignType
@@ -150,19 +155,56 @@ stages:
       targetArchitecture: x86
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - stage: Publish
+  - stage: PrepareForPublish
+    displayName: Prepare for Publish
     dependsOn: Build
     jobs:
-    - template: /eng/jobs/finalize-publish.yml
+    # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing.
+    - template: /eng/jobs/prepare-signed-artifacts.yml
       parameters:
         PublishRidAgnosticPackagesFromJobName: Windows_x64
-
-  - stage: Publish_BAR
-    dependsOn: Publish
-    jobs:
-    # Publish to Build Asset Registry
+    # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact.
     - template: /eng/common/templates/job/publish-build-assets.yml
       parameters:
+        dependsOn: PrepareSignedArtifacts
         pool:
           name: NetCoreInternal-Pool
           queue: buildpool.windows.10.amd64.vs2017
+
+  # Stages-based publishing entry point
+  - template: eng\common\templates\post-build\post-build.yml
+    parameters:
+      dependsOn: [PrepareForPublish]
+      # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871
+      enableSymbolValidation: false
+      # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254
+      enableSigningValidation: false
+      # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604
+      enableSourceLinkValidation: false
+      # Allow symbol publish to emit expected warnings without failing the build.
+      symbolPublishingAdditionalParameters: -warnAsError:$false
+
+  - template: /eng/stages/custom-publish.yml
+    parameters:
+      # Custom publish stage dependency configuration.
+      # - dependsOn: <ID of the Arcade stage that custom publish should run after>
+      #   channel:
+      #     name: <Name of this channel, for logging and readability>
+      #     bar: <Name of the Arcade variable that contains the ID of this channel in BAR>
+      #     storage: <Name of the Latest channel to publish to in dotnetcli blob storage>
+      dependsOnPublishStages:
+      - dependsOn: PublishValidation
+        channel:
+          name: .NET Core 3 Dev
+          bar: PublicDevRelease_30_Channel_Id
+          storage: release/3.0
+      - dependsOn: NetCore_Dev5_PublishValidation
+        channel:
+          name: .NET Core 5 Dev
+          bar: NetCore_5_Dev_Channel_Id
+          storage: master
+      - dependsOn: PVR_PublishValidation
+        channel:
+          name: .NET Tools - Validation
+          bar: PublicValidationRelease_30_Channel_Id
+          storage: dev/validation

eng/Versions.props

@@ -11,8 +11,7 @@
     <DotNetFinalVersionKind Condition="'$(StabilizePackageVersion)' == 'true'">release</DotNetFinalVersionKind>
     <!-- Disable SemVer v2 until after 3.0.0. -->
     <SemanticVersioningV1 Condition="&#xA;      $(MajorVersion) &lt; 3 or&#xA;      '$(MajorVersion).$(MinorVersion).$(PatchVersion)' == '3.0.0'">true</SemanticVersioningV1>
-    <!-- "Latest" channel to publish to. -->
-    <Channel>master</Channel>
+    <!-- Blob storage container that has the "Latest" channel to publish to. -->
     <ContainerName>dotnet</ContainerName>
     <ChecksumContainerName>$(ContainerName)</ChecksumContainerName>
     <!-- Max version of NETCoreApp, used by test projects. -->
@@ -38,7 +37,7 @@
   <PropertyGroup>
     <!-- arcade -->
     <MicrosoftDotNetBuildTasksPackagingPackageVersion>1.0.0-beta.19308.1</MicrosoftDotNetBuildTasksPackagingPackageVersion>
-    <MicrosoftDotNetBuildTasksFeedPackageVersion>2.2.0-beta.19308.1</MicrosoftDotNetBuildTasksFeedPackageVersion>
+    <MicrosoftDotNetBuildTasksFeedPackageVersion>2.2.0-beta.19410.2</MicrosoftDotNetBuildTasksFeedPackageVersion>
     <MicrosoftDotNetVersionToolsTasksPackageVersion>1.0.0-beta.19308.1</MicrosoftDotNetVersionToolsTasksPackageVersion>
     <!-- corefx -->
     <MicrosoftNETCorePlatformsPackageVersion>5.0.0-alpha1.19409.9</MicrosoftNETCorePlatformsPackageVersion>

eng/common/post-build/darc-gather-drop.ps1

@@ -19,7 +19,17 @@ try {
     ExitWithExitCode $exitCode
   }
 
+  # For now, only use a dry run.
+  # Ideally we would change darc to enable a quick request that
+  # would check whether the file exists that you can download it,
+  # and that it won't conflict with other files.
+  # https://github.com/dotnet/arcade/issues/3674
+  # Right now we can't remove continue-on-error because we ocassionally will have
+  # dependencies that have no associated builds (e.g. an old dependency).
+  # We need to add an option to baseline specific dependencies away, or add them manually
+  # to the BAR.
   darc gather-drop --non-shipping `
+    --dry-run `
     --continue-on-error `
     --id $BarBuildId `
     --output-dir $DropLocation `

eng/common/templates/post-build/channels/internal-servicing.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: IS_Publish
@@ -35,6 +37,7 @@ stages:
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job: publish_assets
     displayName: Publish Assets
@@ -92,6 +95,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/channels/netcore-dev-5.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Dev5_Publish
@@ -35,6 +37,7 @@ stages:
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job:
     displayName: Publish Assets
@@ -92,6 +95,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/channels/netcore-tools-latest.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Tools_Latest_Publish
@@ -35,6 +37,7 @@ stages:
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job:
     displayName: Publish Assets
@@ -92,6 +95,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/channels/public-dev-release.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: Publish
@@ -35,6 +37,7 @@ stages:
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job:
     displayName: Publish Assets
@@ -92,6 +95,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/channels/public-release.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: PubRel_Publish
@@ -35,6 +37,7 @@ stages:
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job: publish_assets
     displayName: Publish Assets
@@ -92,6 +95,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/channels/public-validation-release.yml

@@ -1,3 +1,6 @@
+parameters:
+  artifactsPublishingAdditionalParameters: ''
+
 stages:
 - stage: PVR_Publish
   dependsOn: validate
@@ -63,6 +66,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/post-build.yml

@@ -7,6 +7,12 @@ parameters:
     enable: false
     params: ''
 
+  # These parameters let the user customize the call to sdk-task.ps1 for publishing
+  # symbols & general artifacts as well as for signing validation
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  signingValidationAdditionalParameters: ''
+
   # Which stages should finish execution before post-build stages start
   dependsOn: [build]
 
@@ -53,7 +59,8 @@ stages:
             arguments: -task SigningValidation -restore -msbuildEngine dotnet
               /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
               /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
-              /p:Configuration=Release
+              /p:Configuration=Release 
+              ${{ parameters.signingValidationAdditionalParameters }}
 
   - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}:
     - job:
@@ -87,17 +94,31 @@ stages:
 - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\public-dev-release.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\public-validation-release.yml
+  parameters:
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\public-release.yml
+  parameters:
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\internal-servicing.yml
+  parameters:
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}