Merge pull request #3588 from milanlakhani/lano/reportconverter-files

Avoid plist filenames being the same
Ericsson · Feb 7, 2022 · 9004c10 · 9004c10
2 parents 3246665 + 1c36d2f
commit 9004c10
Show file tree

Hide file tree

Showing 28 changed files with 146 additions and 74 deletions.
diff --git a/docs/tools/report-converter.md b/docs/tools/report-converter.md
@@ -84,14 +84,15 @@ optional arguments:
                         file name output of this tool. This tool can produce
                         multiple plist files on the given code analyzer output
                         result file. The problem is if we run this tool
-                        multiple times on the same directory, it may override
-                        some plist files. To prevent this we can generate a
-                        unique hash into the plist file names with this
-                        option. For example: '{source_file}_{analyzer}_xxxxx'.
-                        {source_file} and {analyzer} are special values which
-                        will be replaced with the current analyzer and source
-                        file name where the bug was found. (default:
-                        {source_file}_{analyzer})
+                        multiple times on the same file, it may override some
+                        plist files. To prevent this we can generate a unique
+                        hash into the plist file names with this option. For
+                        example: '{source_file}_{analyzer}_{file_hash}_xxxxx'.
+                        {source_file}, {analyzer} and {file_hash} are special
+                        values which will be replaced with the current
+                        analyzer, source file name and hash of the absolute
+                        file path where the bug was found. (default:
+                        {source_file}_{analyzer}_{file_hash})
   -c, --clean           Delete files stored in the output directory. (default:
                         False)
   -v, --verbose         Set verbosity level. (default: False)

diff --git a/tools/report-converter/codechecker_report_converter/analyzers/analyzer_result.py b/tools/report-converter/codechecker_report_converter/analyzers/analyzer_result.py
@@ -12,6 +12,7 @@
 
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
+import hashlib
 from typing import Dict, List, Optional
 
 from codechecker_report_converter.report import Report, report_file
@@ -39,7 +40,7 @@ def transform(
         analyzer_result_file_path: str,
         output_dir_path: str,
         export_type: str,
-        file_name: str = "{source_file}_{analyzer}",
+        file_name: str = "{source_file}_{analyzer}_{file_hash}",
         metadata: Optional[Dict[str, str]] = None
     ) -> bool:
         """
@@ -143,15 +144,19 @@ def _write(
 
         file_to_report: Dict[str, List[Report]] = defaultdict(list)
         for report in reports:
-            file_to_report[report.file.original_path].append(report)
+            file_path = os.path.normpath(report.file.original_path)
+            file_to_report[file_path].append(report)
 
         analyzer_info = AnalyzerInfo(name=self.TOOL_NAME)
         for file_path, file_reports in file_to_report.items():
             source_file = os.path.basename(file_path)
+            file_hash = hashlib.md5(file_path.encode(errors='ignore')) \
+                .hexdigest()
 
             out_file_name = file_name \
                 .replace("{source_file}", source_file) \
-                .replace("{analyzer}", self.TOOL_NAME)
+                .replace("{analyzer}", self.TOOL_NAME) \
+                .replace("{file_hash}", file_hash)
             out_file_name = f"{out_file_name}.{export_type}"
             out_file_path = os.path.join(output_dir, out_file_name)
 

diff --git a/tools/report-converter/codechecker_report_converter/cli.py b/tools/report-converter/codechecker_report_converter/cli.py
@@ -172,20 +172,22 @@ def __add_arguments_to_parser(parser):
                         type=str,
                         dest='filename',
                         metavar='FILENAME',
-                        default="{source_file}_{analyzer}",
+                        default="{source_file}_{analyzer}_{file_hash}",
                         help="This option can be used to override the default "
                              "plist file name output of this tool. This tool "
                              "can produce multiple plist files on the given "
                              "code analyzer output result file. The problem "
                              "is if we run this tool multiple times on the "
-                             "same directory, it may override some plist "
-                             "files. To prevent this we can generate a unique "
-                             "hash into the plist file names with this "
-                             "option. For example: "
-                             "'{source_file}_{analyzer}_xxxxx'. {source_file} "
-                             "and {analyzer} are special values which will "
-                             "be replaced with the current analyzer and "
-                             "source file name where the bug was found.")
+                             "same file, it may override some plist files. To "
+                             "prevent this we can generate a unique hash into "
+                             "the plist file names with this option. For "
+                             "example: "
+                             "'{source_file}_{analyzer}_{file_hash}_xxxxx'. "
+                             "{source_file}, {analyzer} and {file_hash} are "
+                             "special values which will be replaced with the "
+                             "current analyzer, source file name and hash of "
+                             "the absolute file path where the bug was "
+                             "found. ")
 
     parser.add_argument('-c', '--clean',
                         dest="clean",

diff --git a/tools/report-converter/tests/functional/cmdline/test_cmdline.py b/tools/report-converter/tests/functional/cmdline/test_cmdline.py
@@ -10,6 +10,7 @@
 This module tests the report-converter tool.
 """
 
+import glob
 import json
 import os
 import subprocess
@@ -39,6 +40,9 @@ def test_metadata(self):
                                    'analyzer_command=' + analyzer_command])
             self.assertEqual(0, ret)
 
+            self.assertEqual(
+                len(glob.glob(os.path.join(tmp_dir, '*.plist'))), 2)
+
             metadata_file = os.path.join(tmp_dir, "metadata.json")
             self.assertTrue(os.path.exists(metadata_file))
 

diff --git a/tools/report-converter/tests/functional/cmdline/test_files/Makefile b/tools/report-converter/tests/functional/cmdline/test_files/Makefile
@@ -1,2 +1,2 @@
 all:
-	golint ./files/simple.go > simple.out
+	golint ./files/simple.go ./files/b/simple.go > simple.out
diff --git a/tools/report-converter/tests/functional/cmdline/test_files/files/b/simple.go b/tools/report-converter/tests/functional/cmdline/test_files/files/b/simple.go
@@ -0,0 +1,5 @@
+package pgk
+
+type T2 int
+
+var Y2, Z2 int
diff --git a/tools/report-converter/tests/functional/cmdline/test_files/simple.out b/tools/report-converter/tests/functional/cmdline/test_files/simple.out
@@ -1,2 +1,4 @@
 ./files/simple.go:3:6: exported type T should have comment or be unexported
 ./files/simple.go:5:5: exported var Z should have its own declaration
+./files/b/simple.go:3:6: exported type T2 should have comment or be unexported
+./files/b/simple.go:5:5: exported var Z2 should have its own declaration
diff --git a/tools/report-converter/tests/unit/analyzers/test_asan_parser.py b/tools/report-converter/tests/unit/analyzers/test_asan_parser.py
@@ -52,7 +52,8 @@ def tearDown(self):
     def test_asan(self):
         """ Test for the asan.plist file. """
         self.analyzer_result.transform(
-            'asan.out', self.cc_result_dir, plist.EXTENSION)
+            'asan.out', self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         with open('asan.plist', mode='rb') as pfile:
             exp = plistlib.load(pfile)

diff --git a/tools/report-converter/tests/unit/analyzers/test_clang_tidy_parser.py b/tools/report-converter/tests/unit/analyzers/test_clang_tidy_parser.py
@@ -54,7 +54,8 @@ def __check_analyzer_result(self, analyzer_result, analyzer_result_plist,
                                 source_files, expected_plist):
         """ Check the result of the analyzer transformation. """
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir, analyzer_result_plist)
         with open(plist_file, mode='rb') as pfile:

diff --git a/tools/report-converter/tests/unit/analyzers/test_coccinelle_parser.py b/tools/report-converter/tests/unit/analyzers/test_coccinelle_parser.py
@@ -43,22 +43,25 @@ def test_no_cocci_output_file(self):
                                        'sample.c')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_dir(self):
         """ Test transforming a directory. """
         analyzer_result = os.path.join(self.test_files)
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single output file. """
         analyzer_result = os.path.join(self.test_files, 'sample.out')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'sample.c_coccinelle.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_cppcheck_parser.py b/tools/report-converter/tests/unit/analyzers/test_cppcheck_parser.py
@@ -59,23 +59,26 @@ def test_no_plist_file(self):
                                        'divide_zero.cpp')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_no_plist_dir(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(self.test_files, 'non_existing')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(
             self.test_files, 'out', 'divide_zero.plist')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'divide_zero.cpp_cppcheck.plist')
@@ -99,7 +102,8 @@ def test_transform_directory(self):
         """ Test transforming a directory of plist files. """
         analyzer_result = os.path.join(self.test_files, 'out')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'divide_zero.cpp_cppcheck.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_cpplint_parser.py b/tools/report-converter/tests/unit/analyzers/test_cpplint_parser.py
@@ -42,22 +42,25 @@ def test_no_cpplint_output_file(self):
                                        'sample.cpp')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_dir(self):
         """ Test transforming a directory. """
         analyzer_result = os.path.join(self.test_files)
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single output file. """
         analyzer_result = os.path.join(self.test_files, 'sample.out')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'sample.cpp_cpplint.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_eslint_parser.py b/tools/report-converter/tests/unit/analyzers/test_eslint_parser.py
@@ -42,22 +42,25 @@ def test_no_json_file(self):
                                        'index.js')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_dir(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(self.test_files)
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(self.test_files, 'reports.json')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'index.js_eslint.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_golint_parser.py b/tools/report-converter/tests/unit/analyzers/test_golint_parser.py
@@ -42,22 +42,25 @@ def test_no_go_output_file(self):
                                        'simple.go')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_dir(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(self.test_files)
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single plist file. """
         analyzer_result = os.path.join(self.test_files, 'simple.out')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'simple.go_golint.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_infer_parser.py b/tools/report-converter/tests/unit/analyzers/test_infer_parser.py
@@ -57,7 +57,8 @@ def test_parsing_cpp_res_dir(self):
         analyzer_result = os.path.join(self.test_files, 'infer-out-dead_store')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertTrue(ret)
 
         plist_file = os.path.join(self.cc_result_dir,
@@ -85,7 +86,8 @@ def test_transform_single_cpp_res_file(self):
                                        'report.json')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertTrue(ret)
 
         plist_file = os.path.join(self.cc_result_dir,
@@ -112,7 +114,8 @@ def test_parsing_java_res_dir(self):
                                        'infer-out-null_dereference')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertTrue(ret)
 
         plist_file = os.path.join(self.cc_result_dir,
@@ -141,7 +144,8 @@ def test_transform_single_java_res_file(self):
                                        'report.json')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertTrue(ret)
 
         plist_file = os.path.join(self.cc_result_dir,

diff --git a/tools/report-converter/tests/unit/analyzers/test_kerneldoc_parser.py b/tools/report-converter/tests/unit/analyzers/test_kerneldoc_parser.py
@@ -43,22 +43,25 @@ def test_no_kerneldoc_output_file(self):
                                        'sample.c')
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_dir(self):
         """ Test transforming a directory. """
         analyzer_result = os.path.join(self.test_files)
 
         ret = self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
         self.assertFalse(ret)
 
     def test_transform_single_file(self):
         """ Test transforming single output file. """
         analyzer_result = os.path.join(self.test_files, 'sample.out')
         self.analyzer_result.transform(
-            analyzer_result, self.cc_result_dir, plist.EXTENSION)
+            analyzer_result, self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         plist_file = os.path.join(self.cc_result_dir,
                                   'sample.c_kernel-doc.plist')

diff --git a/tools/report-converter/tests/unit/analyzers/test_lsan_parser.py b/tools/report-converter/tests/unit/analyzers/test_lsan_parser.py
@@ -55,7 +55,8 @@ def tearDown(self):
     def test_san(self):
         """ Test for the lsan.plist file. """
         self.analyzer_result.transform(
-            'lsan.out', self.cc_result_dir, plist.EXTENSION)
+            'lsan.out', self.cc_result_dir, plist.EXTENSION,
+            file_name="{source_file}_{analyzer}")
 
         with open('lsan.plist', mode='rb') as pfile:
             exp = plistlib.load(pfile)