[release/6.0] Port 1ES pool changes to 6.0 (#8408)

* Revert "Revert "Streamline and simplify v3 publishing job"" (#8340) * Revert "Revert "Streamline and simplify v3 publishing job (#8334)" (#8339)" This reverts commit 7189b4a. * Remove parameter usage * Use 1ES pools in templates that run internally (#8344) * Use 1ES pools in templates that run internally - Use the 1ES pools in our templates, jobs, etc. - To reduce some of the cost of this, remove the "Setup Maestro Vars" job and replace with a step so that we don't end up with an additional allocation in those cases. - Call setup maestro vars to populate the azdo params in execute-sdl * Fix DevDiv promotion job (#8370) - Ensure devdiv promotion job uses devdiv pools - Use powershell core (the pools don't have IE initialized so we have to use basic parsing, which is default on powershell core) * Fixup common templates to be compatible with devdiv pools (#8371) In an earlier change, hosted pool usage was moved. This didn't account for differences in pools in devdiv and dnceng. * Fix oneloc template (#8381) The OneLoc template isn't used in the official build when not on main, so validation got skipped.
dotnet · Feb 2, 2022 · cb63eaf · cb63eaf
1 parent 22ebdfe
commit cb63eaf
Show file tree

Hide file tree

Showing 21 changed files with 361 additions and 793 deletions.
diff --git a/azure-pipelines-code-mirror.yml b/azure-pipelines-code-mirror.yml
@@ -10,7 +10,8 @@ jobs:
     jobs:
     - job: Merge_GitHub_to_Azure_DevOps
       pool:
-        vmImage: windows-2019
+        name: NetCore1ESPool-Internal
+        demands: ImageOverride -equals Build.Server.Amd64.VS2019
       variables:
       - name: WorkingDirectoryName
         value: repo-dir

diff --git a/azure-pipelines-codeql.yml b/azure-pipelines-codeql.yml
@@ -23,7 +23,8 @@ stages:
       - job: Windows_NT_CSharp
         timeoutInMinutes: 90
         pool:
-          vmImage: windows-2019
+          name: NetCore1ESPool-Internal
+          demands: ImageOverride -equals Build.Server.Amd64.VS2019
 
         steps:
         - checkout: self

diff --git a/azure-pipelines-merge-mirror.yml b/azure-pipelines-merge-mirror.yml
@@ -10,7 +10,8 @@ jobs:
     jobs:
     - job: Merge_GitHub_to_Azure_DevOps
       pool:
-        vmImage: windows-2019
+        name: NetCore1ESPool-Internal
+        demands: ImageOverride -equals Build.Server.Amd64.VS2019
       variables:
       - name: WorkingDirectoryName
         value: repo-dir

diff --git a/azure-pipelines-weekly.yaml b/azure-pipelines-weekly.yaml
@@ -14,7 +14,9 @@ stages:
     jobs:
     - job: Synchronize
       pool:
-        vmImage: windows-2019
+        name: NetCore1ESPool-Internal
+        demands: ImageOverride -equals Build.Server.Amd64.VS2019
+
       steps:
       - task: UseDotNet@2
         displayName: Install Correct .NET Version

diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
@@ -5,13 +5,8 @@ param(
   [Parameter(Mandatory=$true)][string] $MaestroToken,
   [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
-  [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
-  [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
-  [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
-  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
-  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
-  [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
+  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
 )
 
 try {
@@ -35,27 +30,6 @@ try {
     $optionalParams.Add("--no-wait") | Out-Null
   }
 
-  if ("false" -ne $PublishInstallersAndChecksums) {
-    $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
-  }
-
-  if ("true" -eq $EnableNugetValidation) {
-    $optionalParams.Add("--validate-nuget") | Out-Null
-  }
-
-  if ("true" -eq $EnableSourceLinkValidation) {
-    $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null
-  }
-
-  if ("true" -eq $EnableSigningValidation) {
-    $optionalParams.Add("--validate-signingchecksums") | Out-Null
-
-    if ("" -ne $SigningValidationAdditionalParameters) {
-      $optionalParams.Add("--signing-validation-parameters") | Out-Null
-      $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null
-    }
-  }
-
   & $darc add-build-to-channel `
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `

diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
@@ -29,14 +29,6 @@ parameters:
   # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
   # not pipeline artifacts, so doesn't affect the use of this parameter.
   pipelineArtifactNames: []
-  # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
-  # downloaded from. By default, uses runtime expressions to decide based on the variables set by
-  # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
-  # running without Maestro++/BAR involved, or to download artifacts from a specific existing build
-  # to iterate quickly on SDL changes.
-  AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
-  AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
-  AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
 
 jobs:
 - job: Run_SDL
@@ -60,15 +52,20 @@ jobs:
     - name: GuardianPackagesConfigFile
       value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
   pool:
-    # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
-    ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
-      vmImage: windows-2019
-    ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
-      vmImage: windows-2019
+    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+      name: VSEngSS-MicroBuild2022-1ES
+      demands: Cmd
+    # If it's not devdiv, it's dnceng
+    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+      name: NetCore1ESPool-Internal
+      demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
   - checkout: self
     clean: true
 
+  - template: /eng/common/templates/post-build/setup-maestro-vars.yml
+
   - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
     - ${{ if ne(parameters.artifactNames, '') }}:
       - ${{ each artifactName in parameters.artifactNames }}:

diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml
@@ -3,9 +3,8 @@ parameters:
   dependsOn: ''
 
   # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool:
-    vmImage: windows-2019
-
+  pool: ''
+
   CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
   GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
 
@@ -31,7 +30,18 @@ jobs:
 
   displayName: OneLocBuild
 
-  pool: ${{ parameters.pool }}
+  ${{ if ne(parameters.pool, '') }}:
+    pool: ${{ parameters.pool }}
+  ${{ if eq(parameters.pool, '') }}:
+    pool:
+      # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+        name: VSEngSS-MicroBuild2022-1ES
+        demands: Cmd
+      # If it's not devdiv, it's dnceng
+      ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        name: NetCore1ESPool-Internal
+        demands: ImageOverride -equals Build.Server.Amd64.VS2019
 
   variables:
     - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat

diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
@@ -87,7 +87,15 @@ jobs:
         - ${{ if eq(parameters.enableSourceBuild, true) }}:
           - Source_Build_Complete
         pool:
-          vmImage: windows-2019
+          # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+          ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+            name: VSEngSS-MicroBuild2022-1ES
+            demands: Cmd
+          # If it's not devdiv, it's dnceng
+          ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+            name: NetCore1ESPool-Internal
+            demands: ImageOverride -equals Build.Server.Amd64.VS2019
+
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
         enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml
@@ -4,54 +4,6 @@ variables:
   - group: DotNet-DotNetCli-Storage
   - group: DotNet-MSRC-Storage
   - group: Publish-Build-Assets
-
-  # .NET Core 3.1 Dev
-  - name: PublicDevRelease_31_Channel_Id
-    value: 128
-
-  # .NET 5 Dev
-  - name: Net_5_Dev_Channel_Id
-    value: 131
-
-  # .NET Eng - Validation
-  - name: Net_Eng_Validation_Channel_Id
-    value: 9
-
-  # .NET Eng - Latest
-  - name: Net_Eng_Latest_Channel_Id
-    value: 2
-
-  # .NET 3 Eng - Validation
-  - name: NET_3_Eng_Validation_Channel_Id
-    value: 390
-
-  # .NET 3 Eng
-  - name: NetCore_3_Tools_Channel_Id
-    value: 344
-
-  # .NET Core 3.0 Internal Servicing
-  - name: InternalServicing_30_Channel_Id
-    value: 184
-
-  # .NET Core 3.0 Release
-  - name: PublicRelease_30_Channel_Id
-    value: 19
-
-  # .NET Core 3.1 Release
-  - name: PublicRelease_31_Channel_Id
-    value: 129
-
-  # General Testing
-  - name: GeneralTesting_Channel_Id
-    value: 529
-
-  # .NET Core 3.1 Blazor Features
-  - name: NetCore_31_Blazor_Features_Channel_Id
-    value: 531
-
-  # .NET Core Experimental
-  - name: NetCore_Experimental_Channel_Id
-    value: 562
 
   # Whether the build is internal or not
   - name: IsInternalBuild
@@ -70,26 +22,5 @@ variables:
   - name: SymbolToolVersion
     value: 1.0.1
 
-  # Feed Configurations
-  # These should include the suffix "/index.json"
-
-  # Default locations for Installers and checksums
-  # Public Locations
-  - name: ChecksumsBlobFeedUrl
-    value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
-  - name: InstallersBlobFeedUrl
-    value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
-
-  # Private Locations
-  - name: InternalChecksumsBlobFeedUrl
-    value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
-  - name: InternalChecksumsBlobFeedKey
-    value: $(dotnetclichecksumsmsrc-storage-key)
-
-  - name: InternalInstallersBlobFeedUrl
-    value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
-  - name: InternalInstallersBlobFeedKey
-    value: $(dotnetclimsrc-access-key)
-
   - name: runCodesignValidationInjection
     value: false