Add CodeQL support

Adds a CodeQL plugin that supports CodeQL in the build system.

1. CodeQlBuildPlugin - Generates a CodeQL database for a given build.
2. CodeQlAnalyzePlugin - Analyzes a CodeQL database and interprets
   results.
3. External dependencies - Assist with downloading the CodeQL CLI and
   making it available to the CodeQL plugins.
4. MuCodeQlQueries.qls - A Project Mu CodeQL query set run against
   Project Mu code.
5. Readme.md - A comprehensive readme file to help:
     - Platform integrators understand how to configure the plugin
     - Developers understand how to modify the plugin
     - Users understand how to use the plugin

Read Readme.md for additional details.

CodeQL needs a clean build to form a full database. Therefore, the
plugin previously used the `CleanTree()` method in `UefiBuilder` to
clean the build output directory.

The problem is that the CodeQL plugin runs in pre-build and the
platform build script or other pre-build plugins may have already
placed files in the build output directory they will pick up later
such as during the build or in post-build.

CodeQL does not care about files other than those compiled and linked
in the architecture directories (e.g. "IA32", "X64", etc.). So this
change only removes those directories if they exist and does touch
any other files in the build output directory.

The `query_specifiers` variable should be assigned to the
`STUART_CODEQL_QUERY_SPECIFIERS` environment value if it is present.

- [ ] Breaking change?
- Will this change break pre-existing builds or functionality without
action being taken?
  **No**

Verified on mu_tiano_platforms QemuQ35Pkg.

N/A - Updates internal plugin logic

Signed-off-by: Michael Kubacki <michael.kubacki@microsoft.com>

.pytool/Plugin/CodeQL/CodeQlAnalyzePlugin.py

@@ -0,0 +1,168 @@
+# @file CodeQAnalyzePlugin.py
+#
+# A build plugin that analyzes a CodeQL database.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import json
+import logging
+import os
+import yaml
+
+from common import codeql_plugin
+
+from edk2toolext import edk2_logging
+from edk2toolext.environment.plugintypes.uefi_build_plugin import \
+    IUefiBuildPlugin
+from edk2toolext.environment.uefi_build import UefiBuilder
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
+from edk2toollib.utility_functions import RunCmd
+from pathlib import Path
+
+
+class CodeQlAnalyzePlugin(IUefiBuildPlugin):
+
+    def do_post_build(self, builder: UefiBuilder) -> int:
+        """CodeQL analysis post-build functionality.
+
+        Args:
+            builder (UefiBuilder): A UEFI builder object for this build.
+
+        Returns:
+            int: The number of CodeQL errors found. Zero indicates that
+            AuditOnly mode is enabled or no failures were found.
+        """
+
+        pp = builder.pp.split(os.pathsep)
+        edk2_path = Edk2Path(builder.ws, pp)
+
+        self.builder = builder
+        self.package = edk2_path.GetContainingPackage(
+                            builder.mws.join(builder.ws,
+                                             builder.env.GetValue(
+                                                "ACTIVE_PLATFORM")))
+        self.package_path = Path(
+            edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
+                self.package))
+        self.target = builder.env.GetValue("TARGET")
+
+        self.codeql_db_path = codeql_plugin.get_codeql_db_path(
+                                builder.ws, self.package, self.target,
+                                new_path=False)
+
+        self.codeql_path = codeql_plugin.get_codeql_cli_path()
+        if not self.codeql_path:
+            logging.critical("CodeQL build enabled but CodeQL CLI application "
+                             "not found.")
+            return -1
+
+        codeql_sarif_dir_path = self.codeql_db_path[
+                                        :self.codeql_db_path.rindex('-')]
+        codeql_sarif_dir_path = codeql_sarif_dir_path.replace(
+                                        "-db-", "-analysis-")
+        self.codeql_sarif_path = os.path.join(
+                                        codeql_sarif_dir_path,
+                                        (os.path.basename(
+                                            self.codeql_db_path) +
+                                            ".sarif"))
+
+        edk2_logging.log_progress(f"Analyzing {self.package} ({self.target}) "
+                                  f"CodeQL database at:\n"
+                                  f"           {self.codeql_db_path}")
+        edk2_logging.log_progress(f"Results will be written to:\n"
+                                  f"           {self.codeql_sarif_path}")
+
+        # Packages are allowed to specify package-specific query specifiers
+        # in the package CI YAML file that override the global query specifier.
+        audit_only = False
+        query_specifiers = None
+        package_config_file = Path(os.path.join(
+                                self.package_path, self.package + ".ci.yaml"))
+        if package_config_file.is_file():
+            with open(package_config_file, 'r') as cf:
+                package_config_file_data = yaml.safe_load(cf)
+                if "CodeQlAnalyze" in package_config_file_data:
+                    plugin_data = package_config_file_data["CodeQlAnalyze"]
+                    if "AuditOnly" in plugin_data:
+                        audit_only = plugin_data["AuditOnly"]
+                    if "QuerySpecifiers" in plugin_data:
+                        logging.debug(f"Loading CodeQL query specifiers in "
+                                      f"{str(package_config_file)}")
+                        query_specifiers = plugin_data["QuerySpecifiers"]
+
+        global_audit_only = builder.env.GetValue("STUART_CODEQL_AUDIT_ONLY")
+        if global_audit_only:
+            if global_audit_only.strip().lower() == "true":
+                audit_only = True
+
+        if audit_only:
+            logging.info(f"CodeQL Analyze plugin is in audit only mode for "
+                         f"{self.package} ({self.target}).")
+
+        # Builds can override the query specifiers defined in this plugin
+        # by setting the value in the STUART_CODEQL_QUERY_SPECIFIERS
+        # environment variable.
+        if not query_specifiers:
+            query_specifiers = builder.env.GetValue(
+                                "STUART_CODEQL_QUERY_SPECIFIERS")
+
+        # Use this plugins query set file as the default fallback if it is
+        # not overridden. It is possible the file is not present if modified
+        # locally. In that case, skip the plugin.
+        plugin_query_set = Path(Path(__file__).parent, "MuCodeQlQueries.qls")
+
+        if not query_specifiers and plugin_query_set.is_file():
+            query_specifiers = str(plugin_query_set.resolve())
+
+        if not query_specifiers:
+            logging.warning("Skipping CodeQL analysis since no CodeQL query "
+                            "specifiers were provided.")
+            return 0
+
+        codeql_params = (f'database analyze {self.codeql_db_path} '
+                         f'{query_specifiers} --format=sarifv2.1.0 '
+                         f'--output={self.codeql_sarif_path} --download '
+                         f'--threads=0')
+
+        # CodeQL requires the sarif file parent directory to exist already.
+        Path(self.codeql_sarif_path).parent.mkdir(exist_ok=True, parents=True)
+
+        cmd_ret = RunCmd(self.codeql_path, codeql_params)
+        if cmd_ret != 0:
+            logging.critical(f"CodeQL CLI analysis failed with return code "
+                             f"{cmd_ret}.")
+
+        if not os.path.isfile(self.codeql_sarif_path):
+            logging.critical(f"The sarif file {self.codeql_sarif_path} was "
+                             f"not created. Analysis cannot continue.")
+            return -1
+
+        with open(self.codeql_sarif_path, 'r') as sf:
+            sarif_file_data = json.load(sf)
+
+        try:
+            # Perform minimal JSON parsing to find the number of errors.
+            total_errors = 0
+            for run in sarif_file_data['runs']:
+                total_errors += len(run['results'])
+        except KeyError:
+            logging.critical("Sarif file does not contain expected data. "
+                             "Analysis cannot continue.")
+            return -1
+
+        if total_errors > 0:
+            if audit_only:
+                # Show a warning message so CodeQL analysis is not forgotten.
+                # If the repo owners truly do not want to fix CodeQL issues,
+                # analysis should be disabled entirely.
+                logging.warning(f"{self.package} ({self.target}) CodeQL "
+                                f"analysis ignored {total_errors} errors due "
+                                f"to audit mode being enabled.")
+                return 0
+            else:
+                logging.error(f"{self.package} ({self.target}) CodeQL "
+                              f"analysis failed with {total_errors} errors.")
+
+        return total_errors

.pytool/Plugin/CodeQL/CodeQlAnalyze_plug_in.yaml

@@ -0,0 +1,13 @@
+## @file CodeQlAnalyze_plug_in.py
+#
+# Build plugin used to analyze CodeQL results.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+{
+  "scope": "codeql-analyze",
+  "name": "CodeQL Analyze Plugin",
+  "module": "CodeQlAnalyzePlugin"
+}

.pytool/Plugin/CodeQL/CodeQlBuildPlugin.py

@@ -0,0 +1,170 @@
+# @file CodeQlBuildPlugin.py
+#
+# A build plugin that produces CodeQL results for the present build.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import logging
+import os
+import stat
+from common import codeql_plugin
+from pathlib import Path
+
+from edk2toolext import edk2_logging
+from edk2toolext.environment.plugintypes.uefi_build_plugin import \
+    IUefiBuildPlugin
+from edk2toolext.environment.uefi_build import UefiBuilder
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
+from edk2toollib.utility_functions import GetHostInfo, RemoveTree
+
+
+class CodeQlBuildPlugin(IUefiBuildPlugin):
+
+    def do_pre_build(self, builder: UefiBuilder) -> int:
+        """CodeQL pre-build functionality.
+
+        Args:
+            builder (UefiBuilder): A UEFI builder object for this build.
+
+        Returns:
+            int: The plugin return code. Zero indicates the plugin ran
+            successfully. A non-zero value indicates an unexpected error
+            occurred during plugin execution.
+        """
+
+        pp = builder.pp.split(os.pathsep)
+        edk2_path = Edk2Path(builder.ws, pp)
+
+        self.builder = builder
+        self.package = edk2_path.GetContainingPackage(
+                            builder.mws.join(builder.ws,
+                                             builder.env.GetValue(
+                                                "ACTIVE_PLATFORM")))
+        self.target = builder.env.GetValue("TARGET")
+
+        self.build_output_dir = builder.env.GetValue("BUILD_OUTPUT_BASE")
+
+        self.codeql_db_path = codeql_plugin.get_codeql_db_path(
+                                builder.ws, self.package, self.target)
+
+        edk2_logging.log_progress(f"{self.package} will be built for CodeQL")
+        edk2_logging.log_progress(f"  CodeQL database will be written to "
+                                  f"{self.codeql_db_path}")
+
+        self.codeql_path = codeql_plugin.get_codeql_cli_path()
+        if not self.codeql_path:
+            logging.critical("CodeQL build enabled but CodeQL CLI application "
+                             "not found.")
+            return -1
+
+        # CodeQL can only generate a database on clean build
+        #
+        # Note: builder.CleanTree() cannot be used here as some platforms
+        #       have build steps that run before this plugin that store
+        #       files in the build output directory.
+        #
+        #       CodeQL does not care about with those files or many others such
+        #       as the FV directory, build logs, etc. so instead focus on
+        #       removing only the directories with compilation/linker output
+        #       for the architectures being built (that need clean runs for
+        #       CodeQL to work).
+        targets = self.builder.env.GetValue("TARGET_ARCH").split(" ")
+        for target in targets:
+            directory_to_delete = Path(self.build_output_dir, target)
+
+            if directory_to_delete.is_dir():
+                logging.debug(f"Removing {str(directory_to_delete)} to have a "
+                              f"clean build for CodeQL.")
+                RemoveTree(str(directory_to_delete))
+
+        # A build is required to generate a database
+        builder.SkipBuild = False
+
+        # CodeQL CLI does not handle spaces passed in CLI commands well
+        # (perhaps at all) as discussed here:
+        #   1. https://github.com/github/codeql-cli-binaries/issues/73
+        #   2. https://github.com/github/codeql/issues/4910
+        #
+        # Since it's unclear how quotes are handled and may change in the
+        # future, this code is going to use the workaround to place the
+        # command in an executable file that is instead passed to CodeQL.
+        self.codeql_cmd_path = Path(builder.mws.join(
+                                    builder.ws, self.build_output_dir,
+                                    "codeql_build_command"))
+
+        build_params = self._get_build_params()
+
+        codeql_build_cmd = ""
+        if GetHostInfo().os == "Windows":
+            self.codeql_cmd_path = self.codeql_cmd_path.parent / (
+                self.codeql_cmd_path.name + '.bat')
+        elif GetHostInfo().os == "Linux":
+            self.codeql_cmd_path.suffix = self.codeql_cmd_path.parent / (
+                self.codeql_cmd_path.name + '.sh')
+            codeql_build_cmd += f"#!/bin/bash{os.linesep * 2}"
+        codeql_build_cmd += "build " + build_params
+
+        self.codeql_cmd_path.parent.mkdir(exist_ok=True, parents=True)
+        self.codeql_cmd_path.write_text(encoding='utf8', data=codeql_build_cmd)
+
+        if GetHostInfo().os == "Linux":
+            os.chmod(self.codeql_cmd_path,
+                     os.stat(self.codeql_cmd_path).st_mode | stat.S_IEXEC)
+
+        codeql_params = (f'database create {self.codeql_db_path} '
+                         f'--language=cpp '
+                         f'--source-root={builder.ws} '
+                         f'--command={self.codeql_cmd_path}')
+
+        # Set environment variables so the CodeQL build command is picked up
+        # as the active build command.
+        #
+        # Note: Requires recent changes in edk2-pytool-extensions (0.20.0)
+        #       to support reading these variables.
+        builder.env.SetValue(
+            "EDK_BUILD_CMD", self.codeql_path, "Set in CodeQL Build Plugin")
+        builder.env.SetValue(
+            "EDK_BUILD_PARAMS", codeql_params, "Set in CodeQL Build Plugin")
+
+        return 0
+
+    def _get_build_params(self) -> str:
+        """Returns the build command parameters for this build.
+
+        Based on the well-defined `build` command-line parameters.
+
+        Returns:
+            str: A string representing the parameters for the build command.
+        """
+        build_params = f"-p {self.builder.env.GetValue('ACTIVE_PLATFORM')}"
+        build_params += f" -b {self.target}"
+        build_params += f" -t {self.builder.env.GetValue('TOOL_CHAIN_TAG')}"
+
+        max_threads = self.builder.env.GetValue('MAX_CONCURRENT_THREAD_NUMBER')
+        if max_threads is not None:
+            build_params += f" -n {max_threads}"
+
+        rt = self.builder.env.GetValue("TARGET_ARCH").split(" ")
+        for t in rt:
+            build_params += " -a " + t
+
+        if (self.builder.env.GetValue("BUILDREPORTING") == "TRUE"):
+            build_params += (" -y " +
+                             self.builder.env.GetValue("BUILDREPORT_FILE"))
+            rt = self.builder.env.GetValue("BUILDREPORT_TYPES").split(" ")
+            for t in rt:
+                build_params += " -Y " + t
+
+        # add special processing to handle building a single module
+        mod = self.builder.env.GetValue("BUILDMODULE")
+        if (mod is not None and len(mod.strip()) > 0):
+            build_params += " -m " + mod
+            edk2_logging.log_progress("Single Module Build: " + mod)
+
+        build_vars = self.builder.env.GetAllBuildKeyValues(self.target)
+        for key, value in build_vars.items():
+            build_params += " -D " + key + "=" + value
+
+        return build_params

.pytool/Plugin/CodeQL/CodeQlBuild_plug_in.yaml

@@ -0,0 +1,13 @@
+## @file CodeQlBuild_plug_in.py
+#
+# Build plugin used to produce a CodeQL database from a build.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+{
+  "scope": "codeql-build",
+  "name": "CodeQL Build Plugin",
+  "module": "CodeQlBuildPlugin"
+}