[main] Update dependencies from dotnet/arcade (#7064)

[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -17,19 +17,19 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23580.7">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9cb7d80460fcd74ce5b31633d2deb32a93fd30de</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <!-- Used in repo tooling. Not updated automatically -->
     <Dependency Name="System.Composition" Version="8.0.0-preview.4.23259.5">
       <Uri>https://github.com/dotnet/runtime</Uri>
       <Sha>84a3d0e37e8f22b0b55f8bf932cb788b2bdd728f</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23580.7">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9cb7d80460fcd74ce5b31633d2deb32a93fd30de</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Net.Compilers.Toolset" Version="4.5.0">
       <Uri>https://github.com/dotnet/roslyn</Uri>

eng/common/post-build/redact-logs.ps1

@@ -0,0 +1,81 @@
+[CmdletBinding(PositionalBinding=$False)]
+param(
+  [Parameter(Mandatory=$true, Position=0)][string] $InputPath,
+  [Parameter(Mandatory=$true)][string] $BinlogToolVersion,
+  [Parameter(Mandatory=$false)][string] $DotnetPath,
+  [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json',
+  # File with strings to redact - separated by newlines.
+  #  For comments start the line with '# ' - such lines are ignored 
+  [Parameter(Mandatory=$false)][string] $TokensFilePath,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $packageName = 'binlogtool'
+
+  $dotnet = $DotnetPath
+
+  if (!$dotnet) {
+    $dotnetRoot = InitializeDotNetCli -install:$true
+    $dotnet = "$dotnetRoot\dotnet.exe"
+  }
+
+  $toolList = & "$dotnet" tool list -g
+
+  if ($toolList -like "*$packageName*") {
+    & "$dotnet" tool uninstall $packageName -g
+  }
+
+  $toolPath  = "$PSScriptRoot\..\..\..\.tools"
+  $verbosity = 'minimal'
+
+  New-Item -ItemType Directory -Force -Path $toolPath
+
+  Push-Location -Path $toolPath
+
+  try {
+    Write-Host "Installing Binlog redactor CLI..."
+    Write-Host "'$dotnet' new tool-manifest"
+    & "$dotnet" new tool-manifest
+    Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion"
+    & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion
+
+    if (Test-Path $TokensFilePath) {
+        Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath
+        $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " }
+    }
+
+    $optionalParams = [System.Collections.ArrayList]::new()
+
+    Foreach ($p in $TokensToRedact)
+    {
+      if($p -match '^\$\(.*\)$')
+      {
+        Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable"  -f $p)
+      }          
+      elseif($p)
+      {
+        $optionalParams.Add("-p:" + $p) | Out-Null
+      }
+    }
+
+    & $dotnet binlogtool redact --input:$InputPath --recurse --in-place `
+      @optionalParams
+
+    if ($LastExitCode -ne 0) {
+      Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now."
+    }
+  }
+  finally {
+    Pop-Location
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_"
+  ExitWithExitCode 1
+}

eng/common/templates/job/publish-build-assets.yml

@@ -48,8 +48,8 @@ jobs:
     - group: AzureDevOps-Artifact-Feeds-Pats
     - name: runCodesignValidationInjection
       value: false
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates/post-build/common-variables.yml
+    # unconditional - needed for logs publishing (redactor tool version)
+    - template: /eng/common/templates/post-build/common-variables.yml
 
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)

eng/common/templates/post-build/common-variables.yml

@@ -17,6 +17,8 @@ variables:
     value: 3.0.0
   - name: SymbolToolVersion
     value: 1.0.1
+  - name: BinlogToolVersion
+    value: 1.0.8
 
   - name: runCodesignValidationInjection
     value: false

eng/common/templates/post-build/post-build.yml

@@ -187,6 +187,7 @@ stages:
           parameters:
             StageLabel: 'Validation'
             JobLabel: 'Signing'
+            BinlogToolVersion: $(BinlogToolVersion)
 
     - job:
       displayName: SourceLink Validation

eng/common/templates/steps/publish-logs.yml

@@ -1,6 +1,9 @@
 parameters:
   StageLabel: ''
   JobLabel: ''
+  CustomSensitiveDataList: ''
+  # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed
+  BinlogToolVersion: '1.0.8'
 
 steps:
 - task: Powershell@2
@@ -12,7 +15,30 @@ steps:
       Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
   continueOnError: true
   condition: always()
-
+
+- task: PowerShell@2
+  displayName: Redact Logs
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1
+    # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml
+    # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+    #  If the file exists - sensitive data for redaction will be sourced from it
+    #  (single entry per line, lines starting with '# ' are considered comments and skipped)
+    arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' 
+      -BinlogToolVersion ${{parameters.BinlogToolVersion}}
+      -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+      '$(publishing-dnceng-devdiv-code-r-build-re)'
+      '$(MaestroAccessToken)'
+      '$(dn-bot-all-orgs-artifact-feeds-rw)'
+      '$(akams-client-id)'
+      '$(akams-client-secret)'
+      '$(microsoft-symbol-server-pat)'
+      '$(symweb-symbol-server-pat)'
+      '$(dn-bot-all-orgs-build-rw-code-rw)'
+      ${{parameters.CustomSensitiveDataList}}
+  continueOnError: true
+  condition: always()
+
 - task: PublishBuildArtifacts@1
   displayName: Publish Logs
   inputs:

global.json

@@ -18,6 +18,6 @@
     "rollForward": "patch"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23580.7"
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1"
   }
 }