CodeQL run first pass at implementation (#8279)

Add CodeQL templates and script support.

Adds a template for a CodeQL build, a CodeQL build step, and an arcade implementation of that build that should run weekly.

azure-pipelines-codeql.yml

@@ -0,0 +1,47 @@
+variables:
+  - name: _TeamName
+    value: DotNetCore
+  - group: SDL_Settings
+
+trigger: none
+
+schedules:
+  - cron: 0 12 * * 1
+    displayName: Weekly Monday CodeQL/Semmle run
+    branches:
+      include:
+      - main
+    always: true
+
+stages:
+- stage: build
+  displayName: Build
+  # Three phases for each of the three OSes we want to run on
+  jobs:
+  - template: /eng/common/templates/jobs/codeql-build.yml
+    parameters:
+      jobs:
+      - job: Windows_NT_CSharp
+        timeoutInMinutes: 90
+        pool:
+          vmImage: windows-2019
+
+        steps:
+        - checkout: self
+          clean: true
+
+        - template: /eng/common/templates/steps/execute-codeql.yml
+          parameters:
+            executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
+            buildCommands: 'eng\common\cibuild.cmd -configuration Release -prepareMachine /p:Test=false /p:Sign=false'
+            language: csharp
+            additionalParameters: '-SourceToolsList @("semmle")
+            -TsaInstanceURL $(_TsaInstanceURL)
+            -TsaProjectName $(_TsaProjectName)
+            -TsaNotificationEmail $(_TsaNotificationEmail)
+            -TsaCodebaseAdmin $(_TsaCodebaseAdmin)
+            -TsaBugAreaPath $(_TsaBugAreaPath)
+            -TsaIterationPath $(_TsaIterationPath)
+            -TsaRepositoryName "Arcade"
+            -TsaCodebaseName "Arcade"
+            -TsaPublish $False'

eng/common/sdl/configure-sdl-tool.ps1

@@ -15,7 +15,9 @@ Param(
   # Optional: Additional params to add to any tool using CredScan.
   [string[]] $CrScanAdditionalRunConfigParams,
   # Optional: Additional params to add to any tool using PoliCheck.
-  [string[]] $PoliCheckAdditionalRunConfigParams
+  [string[]] $PoliCheckAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using CodeQL/Semmle.
+  [string[]] $CodeQLAdditionalRunConfigParams
 )
 
 $ErrorActionPreference = 'Stop'
@@ -78,6 +80,11 @@ try {
         $tool.Args += "`"Target < $TargetDirectory`""
       }
       $tool.Args += $PoliCheckAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
+      if ($targetDirectory) {
+        $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+      }
+      $tool.Args += $CodeQLAdditionalRunConfigParams
     }
 
     # Create variable pointing to the args array directly so we can use splat syntax later.

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -34,6 +34,7 @@ Param(
   [string] $GuardianLoggerLevel='Standard',                                                      # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
   [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
   [string[]] $PoliCheckAdditionalRunConfigParams,                                                # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+  [string[]] $CodeQLAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
   [bool] $BreakOnFailure=$False                                                                  # Optional: Fail the build if there were errors during the run
 )
 
@@ -105,7 +106,8 @@ try {
           -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
+          -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
         if ($BreakOnFailure) {
           Exit-IfNZEC "Sdl"
         }

eng/common/templates/jobs/codeql-build.yml

@@ -0,0 +1,31 @@
+parameters:
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+  continueOnError: false
+  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  jobs: []
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+
+jobs:
+- template: /eng/common/templates/jobs/jobs.yml
+  parameters:
+    enableMicrobuild: false
+    enablePublishBuildArtifacts: false
+    enablePublishTestResults: false
+    enablePublishBuildAssets: false
+    enablePublishUsingPipelines: false
+    enableTelemetry: true
+
+    variables:
+      - group: Publish-Build-Assets
+      # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+      # sync with the packages.config file.
+      - name: DefaultGuardianVersion
+        value: 0.109.0
+      - name: GuardianPackagesConfigFile
+        value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+      - name: GuardianVersion
+        value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+
+    jobs: ${{ parameters.jobs }}
+

eng/common/templates/steps/execute-codeql.yml

@@ -0,0 +1,32 @@
+parameters:
+  # Language that should be analyzed. Defaults to csharp
+  language: csharp
+  # Build Commands
+  buildCommands: ''
+  overrideParameters: ''                                       # Optional: to override values for parameters.
+  additionalParameters: ''                                     # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+  # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+  # diagnosis of problems with specific tool configurations.
+  publishGuardianDirectoryToPipeline: false
+  # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+  # parameters rather than relying on YAML. It may be better to use a local script, because you can
+  # reproduce results locally without piecing together a command based on the YAML.
+  executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
+  # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+  # 'continueOnError', the parameter value is not correctly picked up.
+  # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+  # optional: determines whether to continue the build if the step errors;
+  sdlContinueOnError: false
+
+steps:
+- template: /eng/common/templates/steps/execute-sdl.yml
+  parameters:
+    overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
+    executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
+    overrideParameters: ${{ parameters.overrideParameters }}
+    additionalParameters: '${{ parameters.additionalParameters }}
+      -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")'
+    publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
+    sdlContinueOnError: ${{ parameters.sdlContinueOnError }}