diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 141d9f8c362..0fd40014e3e 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -1,13 +1,13 @@
-
+
https://github.com/dotnet/arcade
- fc067a0928f1c8ca4ab3471e9f8edb592e96dec4
+ a188ebf73fb121f0e1d9bc8607c3f5eccd21dfe1
-
+
https://github.com/dotnet/arcade
- fc067a0928f1c8ca4ab3471e9f8edb592e96dec4
+ a188ebf73fb121f0e1d9bc8607c3f5eccd21dfe1
diff --git a/eng/Versions.props b/eng/Versions.props
index 00322aee51b..b2209181477 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -1,6 +1,6 @@
- 6.0.0-beta.21314.1
+ 6.0.0-beta.21363.3
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index a0b5fc37f43..18823840b11 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
+$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
+if ($dotnet6Source -ne $null) {
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
$doc.Save($filename)
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
index 2734601c13c..ad3fb74fd2c 100644
--- a/eng/common/SetupNugetSources.sh
+++ b/eng/common/SetupNugetSources.sh
@@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then
PackageSources+=('dotnet5-internal-transport')
fi
+# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
+grep -i ""
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet6-internal')
+
+ grep -i "" $ConfigFile
+ if [ "$?" != "0" ]; then
+ echo "Adding dotnet6-internal-transport to the packageSources."
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ fi
+ PackageSources+=('dotnet6-internal-transport')
+fi
+
# I want things split line by line
PrevIFS=$IFS
IFS=$'\n'
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 591d8666a84..735a4c82838 100644
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -33,7 +33,6 @@ __AlpinePackages="alpine-base"
__AlpinePackages+=" build-base"
__AlpinePackages+=" linux-headers"
__AlpinePackagesEdgeCommunity=" lldb-dev"
-__AlpinePackagesEdgeMain=" llvm10-libs"
__AlpinePackagesEdgeMain+=" python3"
__AlpinePackagesEdgeMain+=" libedit"
@@ -115,6 +114,8 @@ while :; do
__UbuntuArch=s390x
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+ __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
+ __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package
;;
x86)
@@ -191,6 +192,8 @@ while :; do
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.9
+ __AlpinePackagesEdgeMain+=" llvm11-libs"
+ __AlpinePackagesEdgeMain+=" clang-libs"
;;
alpine3.13)
__CodeName=alpine
@@ -201,6 +204,7 @@ while :; do
__AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain=
+ __AlpinePackages+=" llvm10-libs"
;;
freebsd11)
__FreeBSDBase="11.3-RELEASE"
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
index de348a2e225..25e97ac0077 100644
--- a/eng/common/generate-locproject.ps1
+++ b/eng/common/generate-locproject.ps1
@@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to
# Template files
$jsonFiles = @()
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
+$jsonTemplateFiles | ForEach-Object {
+ $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
+
+ $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
+ $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
$xlfFiles = @()
@@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object {
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}
-$locFiles = $jsonFiles + $xlfFiles
+$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
$locJson = @{
Projects = @(
diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1
new file mode 100644
index 00000000000..4999c307088
--- /dev/null
+++ b/eng/common/sdl/configure-sdl-tool.ps1
@@ -0,0 +1,109 @@
+Param(
+ [string] $GuardianCliLocation,
+ [string] $WorkingDirectory,
+ [string] $TargetDirectory,
+ [string] $GdnFolder,
+ # The list of Guardian tools to configure. For each object in the array:
+ # - If the item is a [hashtable], it must contain these entries:
+ # - Name = The tool name as Guardian knows it.
+ # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
+ # among all tool entries with the same Name.
+ # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
+ # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
+ [object[]] $ToolsList,
+ [string] $GuardianLoggerLevel='Standard',
+ # Optional: Additional params to add to any tool using CredScan.
+ [string[]] $CrScanAdditionalRunConfigParams,
+ # Optional: Additional params to add to any tool using PoliCheck.
+ [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ # Normalize tools list: all in [hashtable] form with defined values for each key.
+ $ToolsList = $ToolsList |
+ ForEach-Object {
+ if ($_ -is [string]) {
+ $_ = @{ Name = $_ }
+ }
+
+ if (-not ($_['Scenario'])) { $_.Scenario = "" }
+ if (-not ($_['Args'])) { $_.Args = @() }
+ $_
+ }
+
+ Write-Host "List of tools to configure:"
+ $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
+
+ # We store config files in the r directory of .gdn
+ $gdnConfigPath = Join-Path $GdnFolder 'r'
+ $ValidPath = Test-Path $GuardianCliLocation
+
+ if ($ValidPath -eq $False)
+ {
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+ ExitWithExitCode 1
+ }
+
+ foreach ($tool in $ToolsList) {
+ # Put together the name and scenario to make a unique key.
+ $toolConfigName = $tool.Name
+ if ($tool.Scenario) {
+ $toolConfigName += "_" + $tool.Scenario
+ }
+
+ Write-Host "=== Configuring $toolConfigName..."
+
+ $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
+
+ # For some tools, add default and automatic args.
+ if ($tool.Name -eq 'credscan') {
+ if ($targetDirectory) {
+ $tool.Args += "TargetDirectory < $TargetDirectory"
+ }
+ $tool.Args += "OutputType < pre"
+ $tool.Args += $CrScanAdditionalRunConfigParams
+ } elseif ($tool.Name -eq 'policheck') {
+ if ($targetDirectory) {
+ $tool.Args += "Target < $TargetDirectory"
+ }
+ $tool.Args += $PoliCheckAdditionalRunConfigParams
+ }
+
+ # Create variable pointing to the args array directly so we can use splat syntax later.
+ $toolArgs = $tool.Args
+
+ # Configure the tool. If args array is provided or the current tool has some default arguments
+ # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
+ # one per parameter. Doc page for "guardian configure":
+ # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
+ Exec-BlockVerbosely {
+ & $GuardianCliLocation configure `
+ --working-directory $WorkingDirectory `
+ --tool $tool.Name `
+ --output-path $gdnConfigFile `
+ --logger-level $GuardianLoggerLevel `
+ --noninteractive `
+ --force `
+ $(if ($toolArgs) { "--args" }) @toolArgs
+ Exit-IfNZEC "Sdl"
+ }
+
+ Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
+ }
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index 2881a56083c..1157151f486 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -7,8 +7,17 @@ Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
- [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
- [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
+
+ # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
+ # format.
+ [object[]] $SourceToolsList,
+ # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
+ # list format.
+ [object[]] $ArtifactToolsList,
+ # Optional: list of SDL tools to run without automatically specifying a target directory. See
+ # 'configure-sdl-tool.ps1' for tools list format.
+ [object[]] $CustomToolsList,
+
[bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
[string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
@@ -63,13 +72,16 @@ try {
ExitWithExitCode 1
}
- & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+ Exec-BlockVerbosely {
+ & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+ }
$gdnFolder = Join-Path $workingDirectory '.gdn'
if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
- Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ Exec-BlockVerbosely {
+ & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ }
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
@@ -80,11 +92,41 @@ try {
}
}
- if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
+ function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
+ if ($tools -and $tools.Count -gt 0) {
+ Exec-BlockVerbosely {
+ & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
+ -GuardianCliLocation $guardianCliLocation `
+ -WorkingDirectory $workingDirectory `
+ -TargetDirectory $targetDirectory `
+ -GdnFolder $gdnFolder `
+ -ToolsList $tools `
+ -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
+ -GuardianLoggerLevel $GuardianLoggerLevel `
+ -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
+ -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ if ($BreakOnFailure) {
+ Exit-IfNZEC "Sdl"
+ }
+ }
+ }
}
- if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+
+ # Configure Artifact and Source tools with default Target directories.
+ Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
+ Configure-ToolsList $SourceToolsList $SourceDirectory
+ # Configure custom tools with no default Target directory.
+ Configure-ToolsList $CustomToolsList $null
+
+ # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
+ # (If we used "run" multiple times, each run would overwrite data from earlier runs.)
+ Exec-BlockVerbosely {
+ & $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
+ -GuardianCliLocation $guardianCliLocation `
+ -WorkingDirectory $workingDirectory `
+ -UpdateBaseline $UpdateBaseline `
+ -GdnFolder $gdnFolder
}
if ($TsaPublish) {
@@ -92,8 +134,9 @@ try {
if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)"
}
- Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ Exec-BlockVerbosely {
+ & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ }
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
@@ -106,7 +149,11 @@ try {
if ($BreakOnFailure) {
Write-Host "Failing the build in case of breaking results..."
- & $guardianCliLocation break
+ Exec-BlockVerbosely {
+ & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+ }
+ } else {
+ Write-Host "Letting the build pass even if there were breaking results..."
}
}
catch {
diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1
new file mode 100644
index 00000000000..68da4fbf257
--- /dev/null
+++ b/eng/common/sdl/extract-artifact-archives.ps1
@@ -0,0 +1,63 @@
+# This script looks for each archive file in a directory and extracts it into the target directory.
+# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
+# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
+param(
+ # Full path to directory where archives are stored.
+ [Parameter(Mandatory=$true)][string] $InputPath,
+ # Full path to directory to extract archives into. May be the same as $InputPath.
+ [Parameter(Mandatory=$true)][string] $ExtractPath
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+try {
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ Measure-Command {
+ $jobs = @()
+
+ # Find archive files for non-Windows and Windows builds.
+ $archiveFiles = @(
+ Get-ChildItem (Join-Path $InputPath "*.tar.gz")
+ Get-ChildItem (Join-Path $InputPath "*.zip")
+ )
+
+ foreach ($targzFile in $archiveFiles) {
+ $jobs += Start-Job -ScriptBlock {
+ $file = $using:targzFile
+ $fileName = [System.IO.Path]::GetFileName($file)
+ $extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
+
+ New-Item $extractDir -ItemType Directory -Force | Out-Null
+
+ Write-Host "Extracting '$file' to '$extractDir'..."
+
+ # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
+ # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
+ # error. Save output so it can be stored in the exception string along with context.
+ $output = tar -xf $file -C $extractDir 2>&1
+ # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
+ # don't have access to the outer scope.
+ if ($LASTEXITCODE -ne 0) {
+ throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
+ }
+
+ Write-Host "Extracted to $extractDir"
+ }
+ }
+
+ Receive-Job $jobs -Wait
+ }
+}
+catch {
+ Write-Host $_
+ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+ ExitWithExitCode 1
+}
diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1
index 3d9c87aba6a..2eac8c78f10 100644
--- a/eng/common/sdl/run-sdl.ps1
+++ b/eng/common/sdl/run-sdl.ps1
@@ -1,13 +1,9 @@
Param(
[string] $GuardianCliLocation,
[string] $WorkingDirectory,
- [string] $TargetDirectory,
[string] $GdnFolder,
- [string[]] $ToolsList,
[string] $UpdateBaseline,
- [string] $GuardianLoggerLevel='Standard',
- [string[]] $CrScanAdditionalRunConfigParams,
- [string[]] $PoliCheckAdditionalRunConfigParams
+ [string] $GuardianLoggerLevel='Standard'
)
$ErrorActionPreference = 'Stop'
@@ -23,7 +19,6 @@ try {
. $PSScriptRoot\..\tools.ps1
# We store config files in the r directory of .gdn
- Write-Host $ToolsList
$gdnConfigPath = Join-Path $GdnFolder 'r'
$ValidPath = Test-Path $GuardianCliLocation
@@ -33,37 +28,18 @@ try {
ExitWithExitCode 1
}
- $configParam = @('--config')
-
- foreach ($tool in $ToolsList) {
- $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
- Write-Host $tool
- # We have to manually configure tools that run on source to look at the source directory only
- if ($tool -eq 'credscan') {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- }
- if ($tool -eq 'policheck') {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- }
-
- $configParam+=$gdnConfigFile
- }
-
- Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
- & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
+ $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
+ Write-Host "Discovered Guardian config files:"
+ $gdnConfigFiles | Out-String | Write-Host
+
+ Exec-BlockVerbosely {
+ & $GuardianCliLocation run `
+ --working-directory $WorkingDirectory `
+ --baseline mainbaseline `
+ --update-baseline $UpdateBaseline `
+ --logger-level $GuardianLoggerLevel `
+ --config @gdnConfigFiles
+ Exit-IfNZEC "Sdl"
}
}
catch {
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index 4a32181fd8f..69eb67849d7 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -2,17 +2,41 @@ parameters:
enable: 'false' # Whether the SDL validation job should execute or not
overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+ # Optional: if specified, restore and use this version of Guardian instead of the default.
+ overrideGuardianVersion: ''
+ # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+ # diagnosis of problems with specific tool configurations.
+ publishGuardianDirectoryToPipeline: false
+ # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+ # parameters rather than relying on YAML. It may be better to use a local script, because you can
+ # reproduce results locally without piecing together a command based on the YAML.
+ executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
# There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
# 'continueOnError', the parameter value is not correctly picked up.
# This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
- downloadArtifacts: true # optional: determines if the artifacts should be dowloaded
+ # optional: determines if build artifacts should be downloaded.
+ downloadArtifacts: true
+ # optional: determines if this job should search the directory of downloaded artifacts for
+ # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks.
+ extractArchiveArtifacts: false
dependsOn: '' # Optional: dependencies of the job
artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
# Usage:
# artifactNames:
# - 'BlobArtifacts'
# - 'Artifacts_Windows_NT_Release'
+ # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
+ # not pipeline artifacts, so doesn't affect the use of this parameter.
+ pipelineArtifactNames: []
+ # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
+ # downloaded from. By default, uses runtime expressions to decide based on the variables set by
+ # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
+ # running without Maestro++/BAR involved, or to download artifacts from a specific existing build
+ # to iterate quickly on SDL changes.
+ AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
jobs:
- job: Run_SDL
@@ -22,16 +46,29 @@ jobs:
variables:
- group: DotNet-VSTS-Bot
- name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ value: ${{ parameters.AzDOProjectName }}
- name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ value: ${{ parameters.AzDOPipelineId }}
- name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ value: ${{ parameters.AzDOBuildId }}
+ # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+ # sync with the packages.config file.
+ - name: DefaultGuardianVersion
+ value: 0.53.3
+ - name: GuardianVersion
+ value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+ - name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
pool:
- name: Hosted VS2017
+ # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
+ ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
+ name: Hosted VS2017
+ ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
+ vmImage: windows-2019
steps:
- checkout: self
clean: true
+
- ${{ if ne(parameters.downloadArtifacts, 'false')}}:
- ${{ if ne(parameters.artifactNames, '') }}:
- ${{ each artifactName in parameters.artifactNames }}:
@@ -59,16 +96,51 @@ jobs:
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
+
+ - ${{ each artifactName in parameters.pipelineArtifactNames }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Pipeline Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: ${{ artifactName }}
+ downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ checkDownloadedFiles: true
+
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
+
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
+
+ - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
+ - powershell: eng/common/sdl/extract-artifact-archives.ps1
+ -InputPath $(Build.ArtifactStagingDirectory)\artifacts
+ -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts
+ displayName: Extract Archive Artifacts
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+
+ - ${{ if ne(parameters.overrideGuardianVersion, '') }}:
+ - powershell: |
+ $content = Get-Content $(GuardianPackagesConfigFile)
+
+ Write-Host "packages.config content was:`n$content"
+
+ $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
+ $content | Set-Content $(GuardianPackagesConfigFile)
+
+ Write-Host "packages.config content updated to:`n$content"
+ displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
+
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
- task: NuGetCommand@2
@@ -79,15 +151,35 @@ jobs:
nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
externalFeedCredentials: GuardianConnect
restoreDirectory: $(Build.SourcesDirectory)\.packages
+
- ${{ if ne(parameters.overrideParameters, '') }}:
- - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
+ - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
+ - powershell: ${{ parameters.executeAllSdlToolsScript }}
+ -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }}
+
+ - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
+ # We want to publish the Guardian results and configuration for easy diagnosis. However, the
+ # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
+ # tooling files. Some of these files are large and aren't useful during an investigation, so
+ # exclude them by simply deleting them before publishing. (As of writing, there is no documented
+ # way to selectively exclude a dir from the pipeline artifact publish task.)
+ - task: DeleteFiles@1
+ displayName: Delete Guardian dependencies to avoid uploading
+ inputs:
+ SourceFolder: $(Agent.BuildDirectory)/.gdn
+ Contents: |
+ c
+ i
+ condition: succeededOrFailed()
+ - publish: $(Agent.BuildDirectory)/.gdn
+ artifact: GuardianConfiguration
+ displayName: Publish GuardianConfiguration
+ condition: succeededOrFailed()
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
index a649d2b5990..b58d42364b9 100644
--- a/eng/common/templates/job/source-index-stage1.yml
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -1,15 +1,19 @@
parameters:
runAsPublic: false
- sourceIndexPackageVersion: 1.0.1-20210421.1
+ sourceIndexPackageVersion: 1.0.1-20210614.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
pool:
vmImage: vs2017-win2016
+ condition: ''
+ dependsOn: ''
jobs:
- job: SourceIndexStage1
+ dependsOn: ${{ parameters.dependsOn }}
+ condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 5619c7aaee1..5d526c74d51 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -42,7 +42,7 @@
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dot.net/v1/dotnet-install.ps1
+# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -106,6 +106,46 @@ function Exec-Process([string]$command, [string]$commandArgs) {
}
}
+# Take the given block, print it, print what the block probably references from the current set of
+# variables using low-effort string matching, then run the block.
+#
+# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes,
+# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less
+# maintainable and less reliable. It is easy to make a mistake and modify the command without
+# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of
+# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in
+# existing source code can also be difficult, because the strings are not aligned to each other and
+# the line may be 300+ columns long.
+#
+# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues.
+#
+# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically.
+# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it
+# doesn't print any info about the variables being used by the command, which is normally the
+# interesting part to diagnose.
+function Exec-BlockVerbosely([scriptblock] $block) {
+ Write-Host "--- Running script block:"
+ $blockString = $block.ToString().Trim()
+ Write-Host $blockString
+
+ Write-Host "--- List of variables that might be used:"
+ # For each variable x in the environment, check the block for a reference to x via simple "$x" or
+ # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x",
+ # among others). It only catches what this function was originally written for: simple
+ # command-line commands.
+ $variableTable = Get-Variable |
+ Where-Object {
+ $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)")
+ } |
+ Format-Table -AutoSize -HideTableHeaders -Wrap |
+ Out-String
+ Write-Host $variableTable.Trim()
+
+ Write-Host "--- Executing:"
+ & $block
+ Write-Host "--- Done running script block!"
+}
+
# createSdkLocationFile parameter enables a file being generated under the toolset directory
# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
# as dot sourcing isn't possible.
@@ -223,7 +263,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
+ $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
Retry({
Write-Host "GET $uri"
@@ -378,7 +418,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
}
$msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
- return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
+
+ $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin"
+ $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false }
+ if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) {
+ $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe"
+ } else {
+ $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe"
+ }
+
+ return $global:_MSBuildExe
}
function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
@@ -623,6 +672,17 @@ function ExitWithExitCode([int] $exitCode) {
exit $exitCode
}
+# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for
+# diagnostics, then exit the script with the $LASTEXITCODE.
+function Exit-IfNZEC([string] $category = "General") {
+ Write-Host "Exit code $LASTEXITCODE"
+ if ($LASTEXITCODE -ne 0) {
+ $message = "Last command failed with exit code $LASTEXITCODE."
+ Write-PipelineTelemetryError -Force -Category $category -Message $message
+ ExitWithExitCode $LASTEXITCODE
+ }
+}
+
function Stop-Processes() {
Write-Host 'Killing running build processes...'
foreach ($processName in $processesToStopOnExit) {
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 05ca99c6b28..828119be411 100644
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true}
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dot.net/v1/dotnet-install.sh
+# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -262,7 +262,7 @@ function with_retries {
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
- local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
+ local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
diff --git a/global.json b/global.json
index 02670883c77..e3b17d8f7f9 100644
--- a/global.json
+++ b/global.json
@@ -4,6 +4,6 @@
"dotnet": "6.0.100-preview.4.21255.9"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21314.1"
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21363.3"
}
}