Make VS16.10 the minimum required VS for builds

azure-pipelines-official.yml

@@ -81,7 +81,7 @@ stages:
     timeoutInMinutes: 360
     pool:
       name: NetCoreInternal-Pool
-      queue: BuildPool.Server.Amd64.VS2017
+      queue: BuildPool.Server.Amd64.VS2019
 
     steps:
     - powershell: Write-Host "##vso[task.setvariable variable=SourceBranchName]$('$(Build.SourceBranch)'.Substring('refs/heads/'.Length))"
@@ -176,6 +176,7 @@ stages:
                    /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                    /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
                    /p:DotnetPublishUsingPipelines=true
+                   /p:IgnoreIbcMergeErrors=true
       condition: succeeded()
 
     - task: PowerShell@2

azure-pipelines-pr-validation.yml

@@ -123,6 +123,7 @@ stages:
                    /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                    /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
                    /p:DotnetPublishUsingPipelines=false
+                   /p:IgnoreIbcMergeErrors=true
                    /p:PreReleaseVersionLabel=pr-validation
       condition: succeeded()
 

azure-pipelines.yml

@@ -155,7 +155,7 @@ jobs:
 - job: Correctness_Determinism
   pool:
     name: NetCorePublic-Pool
-    queue: BuildPool.Windows.10.Amd64.VS2017.Open
+    queue: BuildPool.Windows.10.Amd64.VS2019.Open
   timeoutInMinutes: 90
   steps:
     - template: eng/pipelines/checkout-windows-task.yml
@@ -171,7 +171,7 @@ jobs:
 - job: Correctness_Build
   pool:
     name: NetCorePublic-Pool
-    queue: BuildPool.Windows.10.Amd64.VS2017.Open
+    queue: BuildPool.Windows.10.Amd64.VS2019.Open
   timeoutInMinutes: 90
   steps:
     - template: eng/pipelines/checkout-windows-task.yml

eng/Version.Details.xml

@@ -13,18 +13,18 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21303.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21370.12">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>78da7776965b428ff31da8f1ff2cb073506212b7</Sha>
+      <Sha>89806f0b9e93ad2bbe32c654412835c0801a2032</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.Net.Compilers.Toolset" Version="4.0.0-3.21369.1">
       <Uri>https://github.com/dotnet/roslyn</Uri>
       <Sha>ea623578b108856d3416af28af61060ed3d695e8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21303.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21370.12">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>78da7776965b428ff31da8f1ff2cb073506212b7</Sha>
+      <Sha>89806f0b9e93ad2bbe32c654412835c0801a2032</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -91,7 +91,7 @@
     <MicrosoftDiaSymReaderConverterXmlVersion>1.1.0-beta2-20115-01</MicrosoftDiaSymReaderConverterXmlVersion>
     <MicrosoftDiaSymReaderNativeVersion>16.9.0-beta1.21055.5</MicrosoftDiaSymReaderNativeVersion>
     <MicrosoftDiaSymReaderPortablePdbVersion>1.5.0</MicrosoftDiaSymReaderPortablePdbVersion>
-    <MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21260.1</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>6.0.0-beta.21319.2</MicrosoftDotNetBuildTasksFeedVersion>
     <MicrosoftExtensionsLoggingVersion>5.0.0</MicrosoftExtensionsLoggingVersion>
     <MicrosoftExtensionsLoggingConsoleVersion>5.0.0</MicrosoftExtensionsLoggingConsoleVersion>
     <MicrosoftIdentityModelClientsActiveDirectoryVersion>3.13.8</MicrosoftIdentityModelClientsActiveDirectoryVersion>
@@ -189,7 +189,7 @@
     <MicrosoftVisualStudioWorkspaceVSIntegrationVersion>16.3.43</MicrosoftVisualStudioWorkspaceVSIntegrationVersion>
     <MicrosoftWin32PrimitivesVersion>4.3.0</MicrosoftWin32PrimitivesVersion>
     <MicrosoftWin32RegistryVersion>5.0.0</MicrosoftWin32RegistryVersion>
-    <MSBuildStructuredLoggerVersion>2.1.133</MSBuildStructuredLoggerVersion>
+    <MSBuildStructuredLoggerVersion>2.1.500</MSBuildStructuredLoggerVersion>
     <MDbgVersion>0.1.0</MDbgVersion>
     <MonoOptionsVersion>6.6.0.161</MonoOptionsVersion>
     <MoqVersion>4.10.1</MoqVersion>

eng/common/SetupNugetSources.ps1

@@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
+$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
+if ($dotnet6Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
 $doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet5-internal-transport')
 fi
 
+# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
+grep -i "<add key=\"dotnet6\"" $ConfigFile
+if [ "$?" == "0" ]; then
+    grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal')
+
+    grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal-transport to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal-transport')
+fi
+
 # I want things split line by line
 PrevIFS=$IFS
 IFS=$'\n'

eng/common/cross/build-rootfs.sh

@@ -33,7 +33,6 @@ __AlpinePackages="alpine-base"
 __AlpinePackages+=" build-base"
 __AlpinePackages+=" linux-headers"
 __AlpinePackagesEdgeCommunity=" lldb-dev"
-__AlpinePackagesEdgeMain=" llvm10-libs"
 __AlpinePackagesEdgeMain+=" python3"
 __AlpinePackagesEdgeMain+=" libedit"
 
@@ -115,6 +114,8 @@ while :; do
             __UbuntuArch=s390x
             __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
             __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
             unset __LLDB_Package
             ;;
         x86)
@@ -191,6 +192,8 @@ while :; do
             __CodeName=alpine
             __UbuntuRepo=
             __AlpineVersion=3.9
+            __AlpinePackagesEdgeMain+=" llvm11-libs"
+            __AlpinePackagesEdgeMain+=" clang-libs"
             ;;
         alpine3.13)
             __CodeName=alpine
@@ -201,6 +204,7 @@ while :; do
             __AlpinePackagesEdgeCommunity=
             __AlpinePackages+=$__AlpinePackagesEdgeMain
             __AlpinePackagesEdgeMain=
+            __AlpinePackages+=" llvm10-libs"
             ;;
         freebsd11)
             __FreeBSDBase="11.3-RELEASE"

eng/common/dotnet-install.sh

@@ -70,7 +70,7 @@ case $cpuname in
     ;;
 esac
 
-dotnetRoot="$repo_root/.dotnet"
+dotnetRoot="${repo_root}.dotnet"
 if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
   dotnetRoot="$dotnetRoot/$architecture"
 fi

eng/common/generate-locproject.ps1

@@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to
 
 # Template files
 $jsonFiles = @()
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
+$jsonTemplateFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
+
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
+    $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $xlfFiles = @()
 
@@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object {
     $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
 }
 
-$locFiles = $jsonFiles + $xlfFiles
+$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
 
 $locJson = @{
     Projects = @(

eng/common/internal-feed-operations.ps1

@@ -45,11 +45,11 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  $nugetConfigPath = "$RepoRoot\NuGet.config"
+  $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
 
   if (-Not (Test-Path -Path $nugetConfigPath)) {
     Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
-    ExitWithExitCode 1  
+    ExitWithExitCode 1
   }
 
   $endpoints = New-Object System.Collections.ArrayList
@@ -85,7 +85,7 @@ function SetupCredProvider {
 
 #Workaround for https://github.com/microsoft/msbuild/issues/4430
 function InstallDotNetSdkAndRestoreArcade {
-  $dotnetTempDir = "$RepoRoot\dotnet"
+  $dotnetTempDir = Join-Path $RepoRoot "dotnet"
   $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
   $dotnet = "$dotnetTempDir\dotnet.exe"
   $restoreProjPath = "$PSScriptRoot\restore.proj"

eng/common/internal-feed-operations.sh

@@ -39,7 +39,7 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  local nugetConfigPath="$repo_root/NuGet.config"
+  local nugetConfigPath="{$repo_root}NuGet.config"
 
   if [ ! "$nugetConfigPath" ]; then
     Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"

eng/common/sdk-task.ps1

@@ -34,7 +34,7 @@ function Print-Usage() {
 function Build([string]$target) {
   $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
   $log = Join-Path $LogDir "$task$logSuffix.binlog"
-  $outputPath = Join-Path $ToolsetDir "$task\\"
+  $outputPath = Join-Path $ToolsetDir "$task\"
 
   MSBuild $taskProject `
     /bl:$log `
@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/sdl/configure-sdl-tool.ps1

@@ -0,0 +1,109 @@
+Param(
+  [string] $GuardianCliLocation,
+  [string] $WorkingDirectory,
+  [string] $TargetDirectory,
+  [string] $GdnFolder,
+  # The list of Guardian tools to configure. For each object in the array:
+  # - If the item is a [hashtable], it must contain these entries:
+  #   - Name = The tool name as Guardian knows it.
+  #   - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
+  #     among all tool entries with the same Name.
+  #   - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
+  # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
+  [object[]] $ToolsList,
+  [string] $GuardianLoggerLevel='Standard',
+  # Optional: Additional params to add to any tool using CredScan.
+  [string[]] $CrScanAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using PoliCheck.
+  [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  # Normalize tools list: all in [hashtable] form with defined values for each key.
+  $ToolsList = $ToolsList |
+    ForEach-Object {
+      if ($_ -is [string]) {
+        $_ = @{ Name = $_ }
+      }
+
+      if (-not ($_['Scenario'])) { $_.Scenario = "" }
+      if (-not ($_['Args'])) { $_.Args = @() }
+      $_
+    }
+
+  Write-Host "List of tools to configure:"
+  $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
+
+  # We store config files in the r directory of .gdn
+  $gdnConfigPath = Join-Path $GdnFolder 'r'
+  $ValidPath = Test-Path $GuardianCliLocation
+
+  if ($ValidPath -eq $False)
+  {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+    ExitWithExitCode 1
+  }
+
+  foreach ($tool in $ToolsList) {
+    # Put together the name and scenario to make a unique key.
+    $toolConfigName = $tool.Name
+    if ($tool.Scenario) {
+      $toolConfigName += "_" + $tool.Scenario
+    }
+
+    Write-Host "=== Configuring $toolConfigName..."
+
+    $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
+
+    # For some tools, add default and automatic args.
+    if ($tool.Name -eq 'credscan') {
+      if ($targetDirectory) {
+        $tool.Args += "TargetDirectory < $TargetDirectory"
+      }
+      $tool.Args += "OutputType < pre"
+      $tool.Args += $CrScanAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'policheck') {
+      if ($targetDirectory) {
+        $tool.Args += "Target < $TargetDirectory"
+      }
+      $tool.Args += $PoliCheckAdditionalRunConfigParams
+    }
+
+    # Create variable pointing to the args array directly so we can use splat syntax later.
+    $toolArgs = $tool.Args
+
+    # Configure the tool. If args array is provided or the current tool has some default arguments
+    # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
+    # one per parameter. Doc page for "guardian configure":
+    # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
+    Exec-BlockVerbosely {
+      & $GuardianCliLocation configure `
+        --working-directory $WorkingDirectory `
+        --tool $tool.Name `
+        --output-path $gdnConfigFile `
+        --logger-level $GuardianLoggerLevel `
+        --noninteractive `
+        --force `
+        $(if ($toolArgs) { "--args" }) @toolArgs
+      Exit-IfNZEC "Sdl"
+    }
+
+    Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}