[cli][server] Refactoring code for sarif support

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
         run: make pylint pycodestyle
 
   tools:
-    name: Tools (plist-to-html, report-converter, etc.)
+    name: Tools (report-converter, etc.)
     runs-on: ubuntu-18.04
 
     steps:
@@ -57,18 +57,6 @@ jobs:
           pip install -r requirements_py/dev/requirements.txt
           make test
 
-      - name: Run codechecker-report-hash tests
-        working-directory: tools/codechecker_report_hash
-        run: |
-          pip install -r requirements_py/dev/requirements.txt
-          make test
-
-      - name: Run plist-to-html tests
-        working-directory: tools/plist_to_html
-        run: |
-          pip install -r requirements_py/dev/requirements.txt
-          make test
-
       - name: Run report-converter tests
         working-directory: tools/report-converter
         run: |
@@ -129,13 +117,9 @@ jobs:
         run: |
           pip install -r requirements_py/dev/requirements.txt
 
-      - name: Run unit tests
-        working-directory: codechecker_common/tests
-        run: make -C unit test_unit
-
-      - name: Run unit tests coverage
+      - name: Run mypy tests
         working-directory: codechecker_common/tests
-        run: make -C unit test_unit_cov
+        run: make mypy
 
   web:
     name: Web

.gitmessage

@@ -18,7 +18,6 @@
 #    gui           (Changes in the web GUI)
 #    fix           (Bug fix)
 #    feat          (New feature)
-#    plist2html    (Changes in the plist-to-html tool)
 #    refactor      (Refactoring code)
 #    server        (Server related changes)
 #    style         (Formatting, missing semi colons, etc; no code change)

CONTRIBUTING.md

@@ -129,7 +129,7 @@ CodeChecker, gerrit integration, debug etc.
 
 #### `tools`
 This folder contains tools which are used by the `analyzer` and `web` part
-of the CodeChecker such as `plist-to-html` and `tu_collector`.
+of the CodeChecker such as `tu_collector`.
 
 #### `web`
 This folder contains source code of the CodeChecker web server and web client.

Makefile

@@ -112,8 +112,7 @@ clean_venv:
 	rm -rf venv
 
 PIP_DEV_DEPS_CMD = make -C $(CC_ANALYZER) pip_dev_deps && \
-  make -C $(CC_WEB) pip_dev_deps && \
-  make -C $(CC_TOOLS)/plist_to_html pip_dev_deps
+  make -C $(CC_WEB) pip_dev_deps
 
 pip_dev_deps:
 	# Install the depencies for analyze, web and the tools.
@@ -128,27 +127,20 @@ clean_venv_dev:
 	rm -rf venv_dev
 	$(MAKE) -C $(CC_ANALYZER) clean_venv_dev
 	$(MAKE) -C $(CC_WEB) clean_venv_dev
-	$(MAKE) -C $(CC_TOOLS)/plist_to_html clean_venv_dev
 
 clean: clean_package
 	$(MAKE) -C $(CC_WEB) clean
 
-clean_package: clean_plist_to_html clean_tu_collector clean_report_converter clean_report_hash clean_statistics_collector
+clean_package: clean_tu_collector clean_report_converter clean_statistics_collector
 	rm -rf $(BUILD_DIR)
 	find . -name "*.pyc" -delete
 
-clean_plist_to_html:
-	$(MAKE) -C $(CC_TOOLS)/plist_to_html clean
-
 clean_tu_collector:
 	$(MAKE) -C $(CC_TOOLS)/tu_collector clean
 
 clean_report_converter:
 	$(MAKE) -C $(CC_TOOLS)/report-converter clean
 
-clean_report_hash:
-	$(MAKE) -C $(CC_TOOLS)/codechecker_report_hash clean
-
 clean_statistics_collector:
 	$(MAKE) -C $(CC_ANALYZER_TOOLS)/statistics_collector clean
 
@@ -168,23 +160,17 @@ pylint_in_env: venv_dev
 
 PYCODE_CMD = $(MAKE) -C $(CC_ANALYZER) pycodestyle && \
 	$(MAKE) -C $(CC_WEB) pycodestyle && \
-	pycodestyle bin codechecker_common scripts
+	pycodestyle bin codechecker_common
 
 pycodestyle:
 	$(PYCODE_CMD)
 
 pycodestyle_in_env:
 	$(ACTIVATE_DEV_VENV) && $(PYCODE_CMD)
 
-test: test_common test_analyzer test_web
-
-test_in_env: test_common_in_env test_analyzer_in_env test_web_in_env
-
-test_common:
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_COMMON)/tests/unit test_unit
+test: test_analyzer test_web
 
-test_common_in_env:
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_COMMON)/tests/unit test_unit_in_env
+test_in_env: test_analyzer_in_env test_web_in_env
 
 test_analyzer:
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_ANALYZER) test
@@ -221,12 +207,10 @@ test_web_in_env:
 test_unit:
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_ANALYZER) test_unit
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB) test_unit
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_COMMON)/tests/unit test_unit
 
 test_unit_cov:
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_ANALYZER) test_unit_cov
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB) test_unit_cov
-	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_COMMON)/tests/unit test_unit_cov
 
 test_unit_in_env:
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_ANALYZER) test_unit_in_env

analyzer/Makefile

@@ -50,13 +50,6 @@ package_dir_structure:
 	mkdir -p $(CC_BUILD_BIN_DIR) && \
 	mkdir -p $(CC_BUILD_LIB_DIR)
 
-build_plist_to_html:
-	$(MAKE) -C $(ROOT)/tools/plist_to_html build
-
-package_plist_to_html: build_plist_to_html package_dir_structure
-	# Copy plist-to-html files.
-	cp -r $(CC_TOOLS)/plist_to_html/build/plist_to_html/plist_to_html $(CC_BUILD_LIB_DIR)
-
 build_tu_collector:
 	$(MAKE) -C $(ROOT)/tools/tu_collector build
 
@@ -67,12 +60,6 @@ package_tu_collector: build_tu_collector package_dir_structure
 	cd $(CC_BUILD_BIN_DIR) && \
 	ln -sf ../lib/python3/tu_collector/tu_collector.py tu_collector
 
-build_report_hash:
-	$(MAKE) -C $(CC_TOOLS)/codechecker_report_hash build
-
-package_report_hash: build_report_hash package_dir_structure
-	cp -rp $(CC_TOOLS)/codechecker_report_hash/build/codechecker_report_hash/codechecker_report_hash $(CC_BUILD_LIB_DIR)
-
 build_report_converter:
 	$(MAKE) -C $(CC_TOOLS)/report-converter build
 
@@ -86,7 +73,7 @@ build_merge_clang_extdef_mappings:
 	$(MAKE) -C $(CC_ANALYZER)/tools/merge_clang_extdef_mappings build
 
 package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure
-	# Copy plist-to-html files.
+	# Copy files.
 	cp -r tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \
 	chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \
 	cd $(CC_BUILD_BIN_DIR) && \
@@ -113,9 +100,8 @@ package_bazel_compile_commands: build_bazel_compile_commands package_dir_structu
 	ln -sf ../lib/python3/bazel_compile_commands/bazel_compile_commands.py bazel-compile-commands
 
 # This target should be used from the top level Makefile to build the package
-# together with the web part. This way we will not build plist-to-html
-# multiple times.
-package_analyzer: package_dir_structure package_plist_to_html package_tu_collector package_report_hash package_merge_clang_extdef_mappings package_report_converter package_statistics_collector package_bazel_compile_commands
+# together with the web part. This way we will not build tools multiple times.
+package_analyzer: package_dir_structure package_tu_collector package_merge_clang_extdef_mappings package_report_converter package_statistics_collector package_bazel_compile_commands
 
 package: package_analyzer
 	# Copy libraries.
@@ -182,13 +168,10 @@ else
   endif
 endif
 
-clean_package: clean_plist_to_html clean_tu_collector clean_bazel_compile_commands
+clean_package: clean_tu_collector clean_bazel_compile_commands
 	rm -rf $(BUILD_DIR)
 	find . -name "*.pyc" -delete
 
-clean_plist_to_html:
-	rm -rf $(ROOT)/tools/plist_to_html/build
-
 clean_tu_collector:
 	rm -rf $(ROOT)/tools/tu_collector/build
 

analyzer/codechecker_analyzer/analysis_manager.py

@@ -25,7 +25,6 @@
 import psutil
 
 from codechecker_analyzer import env
-from codechecker_common import plist_parser
 from codechecker_common.logger import get_logger
 
 from codechecker_statistics_collector.collectors.special_return_value import \
@@ -233,18 +232,13 @@ def handle_success(rh, result_file, result_base, skip_handler,
         save_output(os.path.join(success_dir, result_base),
                     rh.analyzer_stdout, rh.analyzer_stderr)
 
-    rh.postprocess_result()
+    rh.postprocess_result(skip_handler)
+
     # Generated reports will be handled separately at store.
 
     save_metadata(result_file, rh.analyzer_result_file,
                   rh.analyzed_source_file)
 
-    if skip_handler:
-        # We need to check the plist content because skipping
-        # reports in headers can be done only this way.
-        plist_parser.skip_report_from_plist(result_file,
-                                            skip_handler)
-
 
 def handle_reproducer(source_analyzer, rh, zip_file, actions_map):
     """
@@ -321,7 +315,9 @@ def handle_reproducer(source_analyzer, rh, zip_file, actions_map):
     LOG.debug("ZIP file written at '%s'", zip_file)
 
 
-def handle_failure(source_analyzer, rh, zip_file, result_base, actions_map):
+def handle_failure(
+    source_analyzer, rh, zip_file, result_base, actions_map, skip_handler
+):
     """
     If the analysis fails a debug zip is packed together which contains
     build, analysis information and source files to be able to
@@ -335,7 +331,7 @@ def handle_failure(source_analyzer, rh, zip_file, result_base, actions_map):
     checks = source_analyzer.config_handler.checks()
     state = checks.get('clang-diagnostic-error', (CheckerState.default, ''))[0]
     if state != CheckerState.disabled:
-        rh.postprocess_result()
+        rh.postprocess_result(skip_handler)
 
     # Remove files that successfully analyzed earlier on.
     plist_file = result_base + ".plist"
@@ -611,7 +607,7 @@ def handle_analysis_result(success, zip_file=zip_file):
             elif not generate_reproducer:
                 handle_failure(source_analyzer, rh,
                                os.path.join(failed_dir, zip_file),
-                               result_base, actions_map)
+                               result_base, actions_map, skip_handler)
 
         if rh.analyzer_returncode == 0:
             handle_analysis_result(success=True)
@@ -679,12 +675,6 @@ def handle_analysis_result(success, zip_file=zip_file):
         collect_ctu_involved_files(rh, source_analyzer,
                                    output_dirs['ctu_connections'])
 
-        if skip_handler and os.path.exists(result_file):
-            # We need to check the plist content because skipping
-            # reports in headers can be done only this way.
-            plist_parser.skip_report_from_plist(result_file,
-                                                skip_handler)
-
         if not quiet_output_on_stdout:
             if rh.analyzer_returncode:
                 LOG.error('\n%s', rh.analyzer_stdout)

analyzer/codechecker_analyzer/analyzer_context.py

@@ -16,10 +16,11 @@
 import os
 import sys
 
+from codechecker_report_converter.util import load_json_or_empty
+
 from codechecker_common import logger
 from codechecker_common.checker_labels import CheckerLabels
 from codechecker_common.singleton import Singleton
-from codechecker_common.util import load_json_or_empty
 
 from . import env
 
@@ -231,7 +232,8 @@ def logger_lib_name(self):
 
     @property
     def path_plist_to_html_dist(self):
-        return os.path.join(self._lib_dir_path, 'plist_to_html', 'static')
+        return os.path.join(self._lib_dir_path, 'codechecker_report_converter',
+                            'report', 'output', 'html', 'static')
 
     @property
     def path_env_extra(self):

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -30,7 +30,7 @@
 from . import config_handler
 from . import ctu_triple_arch
 from . import version
-from .result_handler import ResultHandlerClangSA
+from .result_handler import ClangSAResultHandler
 
 LOG = get_logger('analyzer')
 
@@ -410,7 +410,7 @@ def construct_result_handler(self, buildaction, report_output,
         """
         See base class for docs.
         """
-        res_handler = ResultHandlerClangSA(buildaction, report_output,
+        res_handler = ClangSAResultHandler(buildaction, report_output,
                                            self.config_handler.report_hash)
 
         res_handler.checker_labels = checker_labels

analyzer/codechecker_analyzer/analyzers/clangsa/result_handler.py

@@ -11,30 +11,49 @@
 
 import os
 
+from typing import Optional
+
+from codechecker_report_converter.report.parser.base import AnalyzerInfo
+from codechecker_report_converter.report import report_file
+from codechecker_report_converter.report.hash import get_report_hash, HashType
 from codechecker_common.logger import get_logger
-from codechecker_report_hash.hash import HashType, replace_report_hash
+from codechecker_common.skiplist_handler import SkipListHandler
 
 from ..result_handler_base import ResultHandler
 
 LOG = get_logger('report')
 
 
-class ResultHandlerClangSA(ResultHandler):
+class ClangSAResultHandler(ResultHandler):
     """
-    Use context free hash if enabled.
+    Create analyzer result file for Clang Static Analyzer output.
     """
 
-    def postprocess_result(self):
+    def __init__(self, *args, **kwargs):
+        self.analyzer_info = AnalyzerInfo(name='clangsa')
+
+        super(ClangSAResultHandler, self).__init__(*args, **kwargs)
+
+    def postprocess_result(self, skip_handler: Optional[SkipListHandler]):
         """
-        Override the context sensitive issue hash in the plist files to
-        context insensitive if it is enabled during analysis.
+        Generate analyzer result output file which can be parsed and stored
+        into the database.
         """
         if os.path.exists(self.analyzer_result_file):
+            reports = report_file.get_reports(
+                self.analyzer_result_file, self.checker_labels)
+            reports = [r for r in reports if not r.skip(skip_handler)]
+
+            hash_type = None
             if self.report_hash_type in ['context-free', 'context-free-v2']:
-                replace_report_hash(
-                    self.analyzer_result_file,
-                    HashType.CONTEXT_FREE)
+                hash_type = HashType.CONTEXT_FREE
             elif self.report_hash_type == 'diagnostic-message':
-                replace_report_hash(
-                    self.analyzer_result_file,
-                    HashType.DIAGNOSTIC_MESSAGE)
+                hash_type = HashType.DIAGNOSTIC_MESSAGE
+
+            if hash_type is not None:
+                for report in reports:
+                    report.report_hash = get_report_hash(report, hash_type)
+
+            report_file.create(
+                self.analyzer_result_file, reports, self.checker_labels,
+                self.analyzer_info)

analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py

@@ -327,9 +327,8 @@ def construct_result_handler(self, buildaction, report_output,
         See base class for docs.
         """
         report_hash = self.config_handler.report_hash
-        res_handler = result_handler.ClangTidyPlistToFile(buildaction,
-                                                          report_output,
-                                                          report_hash)
+        res_handler = result_handler.ClangTidyResultHandler(
+            buildaction, report_output, report_hash)
 
         res_handler.checker_labels = checker_labels
         res_handler.skiplist_handler = skiplist_handler