Merge pull request #3017 from jay24rajput/sphinx_parser

[report-converter] Sphinx parser

docs/README.md

@@ -118,6 +118,7 @@ The following tools are supported:
 | **TypeScript** | [TSLint](/tools/report-converter/README.md#tslint)    |
 | **Go**         | [Golint](/tools/report-converter/README.md#golint)    |
 | **Markdown**   | [Markdownlint](/tools/report-converter/README.md#markdownlint)    |
+|                | [Sphinx](/tools/report-converter/Readme.md#sphinx)    |
 
 
 For details see 

docs/supported_code_analyzers.md

@@ -33,6 +33,7 @@ CodeChecker result directory which can be stored to a CodeChecker server.
 |                | [Staticcheck](https://staticcheck.io/)    | ✗ |
 |                | [go-critic](https://github.com/go-critic/go-critic)    | ✗ |
 | **Markdown**   | [Markdownlint](https://github.com/markdownlint/markdownlint)    | ✓ |
+|                | [Sphinx](https://github.com/sphinx-doc/sphinx)    | ✓ |
 
 ## Clang Sanitizers
 | Name         | Support storage of analyzer results |

tools/report-converter/README.md

@@ -23,6 +23,7 @@ a CodeChecker server.
 * [Coccinelle](#coccinelle)
 * [Smatch](#smatch)
 * [Kernel-Doc](#kernel-doc)
+* [Sphinx](#sphinx)
 * [License](#license)
 
 ## Install guide
@@ -57,7 +58,7 @@ optional arguments:
                         Currently supported output types are: asan, clang-
                         tidy, coccinelle, cppcheck, eslint, fbinfer, golint,
                         kernel-doc, msan, pyflakes, pylint, smatch, spotbugs, 
-                        tsan, tslint, ubsan.
+                        sphinx, tsan, tslint, ubsan.
   --meta [META [META ...]]
                         Metadata information which will be stored alongside
                         the run when the created report directory will be
@@ -95,6 +96,7 @@ Supported analyzers:
   pylint - Pylint, https://www.pylint.org
   smatch - smatch, https://repo.or.cz/w/smatch.git
   spotbugs - spotbugs, https://spotbugs.github.io
+  sphinx - sphinx, https://github.com/sphinx-doc/sphinx
   tsan - ThreadSanitizer, https://clang.llvm.org/docs/ThreadSanitizer.html
   tslint - TSLint, https://palantir.github.io/tslint
   ubsan - UndefinedBehaviorSanitizer, https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
@@ -465,12 +467,40 @@ make htmldocs 2>&1 | tee kernel-docs.out
 
 # Use 'report-converter' to create a CodeChecker report directory from the
 # analyzer result of Kernel-Doc
-report-converter -t kernel-doc -o ./codechecker_kernel_doc_reports ./sphinx_output.out
+report-converter -t kernel-doc -o ./codechecker_kernel_doc_reports ./kernel-docs.out
 
 # Store the Kernel-Doc reports with CodeChecker.
 CodeChecker store ./codechecker_kernel_doc_reports -n kernel-doc
 ```
 
+## [Sphinx](https://github.com/sphinx-doc/sphinx)
+[Sphinx](https://github.com/sphinx-doc/sphinx) Sphinx is a documentation generator 
+or a tool that translates a set of plain text source files into various output formats, 
+automatically producing cross-references, indices, etc.
+
+The recommended way of running Sphinx is to redirect the output to a file and
+give this file to the report converter tool.
+
+The following example shows you how to run Sphinx on kernel sources 
+and store the results found by Sphinx to the CodeChecker database.
+
+```sh
+# Change Directory to your project
+cd path/to/linux/kernel/repository
+
+# Run Sphinx
+# Note: The output of the following command will be both of sphinx and kernel-doc, 
+# but the parser will parse only sphinx output
+make htmldocs 2>&1 | tee sphinx.out
+
+# Use 'report-converter' to create a CodeChecker report directory from the
+# analyzer result of Sphinx
+report-converter -t sphinx -o ./codechecker_sphinx_reports ./sphinx.out
+
+# Store the Sphinx reports with CodeChecker.
+CodeChecker store ./codechecker_sphinx_reports -n sphinx
+```
+
 ## License
 
 The project is licensed under Apache License v2.0 with LLVM Exceptions.

tools/report-converter/codechecker_report_converter/cli.py

@@ -58,6 +58,8 @@
     SmatchAnalyzerResult  # noqa
 from codechecker_report_converter.kerneldoc.analyzer_result import \
     KernelDocAnalyzerResult  # noqa
+from codechecker_report_converter.sphinx.analyzer_result import \
+    SphinxAnalyzerResult  # noqa
 
 
 LOG = logging.getLogger('ReportConverter')
@@ -95,7 +97,8 @@ class RawDescriptionDefaultHelpFormatter(
     MarkdownlintAnalyzerResult.TOOL_NAME: MarkdownlintAnalyzerResult,
     CoccinelleAnalyzerResult.TOOL_NAME: CoccinelleAnalyzerResult,
     SmatchAnalyzerResult.TOOL_NAME: SmatchAnalyzerResult,
-    KernelDocAnalyzerResult.TOOL_NAME: KernelDocAnalyzerResult
+    KernelDocAnalyzerResult.TOOL_NAME: KernelDocAnalyzerResult,
+    SphinxAnalyzerResult.TOOL_NAME: SphinxAnalyzerResult
 }
 
 supported_metadata_keys = ["analyzer_command", "analyzer_version"]

tools/report-converter/codechecker_report_converter/sphinx/__init__.py

@@ -0,0 +1,7 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------

tools/report-converter/codechecker_report_converter/sphinx/analyzer_result.py

@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+from codechecker_report_converter.analyzer_result import AnalyzerResult
+
+from .output_parser import SphinxParser
+from ..plist_converter import PlistConverter
+
+
+class SphinxAnalyzerResult(AnalyzerResult):
+    """ Transform analyzer result of Sphinx. """
+
+    TOOL_NAME = 'sphinx'
+    NAME = 'Sphinx'
+    URL = 'https://github.com/sphinx-doc/sphinx'
+
+    def parse(self, analyzer_result):
+        """ Creates plist files from the given analyzer result to the given
+        output directory.
+        """
+        parser = SphinxParser(analyzer_result)
+
+        content = self._get_analyzer_result_file_content(analyzer_result)
+        if not content:
+            return
+
+        messages = parser.parse_messages(content)
+
+        plist_converter = PlistConverter(self.TOOL_NAME)
+        plist_converter.add_messages(messages)
+        return plist_converter.get_plist_results()

tools/report-converter/codechecker_report_converter/sphinx/output_parser.py

@@ -0,0 +1,61 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+import logging
+import os
+import re
+
+from ..output_parser import BaseParser, Message
+LOG = logging.getLogger('ReportConverter')
+
+
+class SphinxParser(BaseParser):
+    """
+    Parser for Sphinx Output
+    """
+
+    def __init__(self, analyzer_result):
+        super(SphinxParser, self).__init__()
+
+        self.analyzer_result = analyzer_result
+
+        self.message_line_re = re.compile(
+            # File path starting with '/' and followed by a ':'.
+            r'^(?P<path>\/[\S ]+?):'
+            # Line number followed by a ':'.
+            r'(?P<line>\d+?):\s'
+            # Message.
+            r'(?P<message>[\S \t]+)\s*')
+
+    def parse_message(self, it, line):
+        """
+        Actual Parsing function for the given line
+        It is expected that each line contains a seperate report
+        """
+        match = self.message_line_re.match(line)
+
+        if match is None:
+            return None, next(it)
+
+        file_path = os.path.normpath(
+            os.path.join(os.path.dirname(self.analyzer_result),
+                         match.group('path')))
+
+        checker_name = None
+
+        message = Message(
+            file_path,
+            int(match.group('line')),
+            0,
+            match.group('message').strip(),
+            checker_name)
+
+        try:
+            return message, next(it)
+        except StopIteration:
+            return message, ''

tools/report-converter/tests/unit/sphinx_output_test_files/files/sample.rst

@@ -0,0 +1,4 @@
+Sample Test File
+----------------------------------
+
+This is a sample test file for testing the output of sphinx parser

tools/report-converter/tests/unit/sphinx_output_test_files/sample.expected.plist

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>diagnostics</key>
+	<array>
+		<dict>
+			<key>category</key>
+			<string>unknown</string>
+			<key>check_name</key>
+			<string>sphinx</string>
+			<key>description</key>
+			<string>WARNING: Title underline too short.</string>
+			<key>issue_hash_content_of_line_in_context</key>
+			<string>cb9e277095a47d920ef5a846b3af91fa</string>
+			<key>location</key>
+			<dict>
+				<key>col</key>
+				<integer>0</integer>
+				<key>file</key>
+				<integer>0</integer>
+				<key>line</key>
+				<integer>2</integer>
+			</dict>
+			<key>path</key>
+			<array>
+				<dict>
+					<key>depth</key>
+					<integer>0</integer>
+					<key>kind</key>
+					<string>event</string>
+					<key>location</key>
+					<dict>
+						<key>col</key>
+						<integer>0</integer>
+						<key>file</key>
+						<integer>0</integer>
+						<key>line</key>
+						<integer>2</integer>
+					</dict>
+					<key>message</key>
+					<string>WARNING: Title underline too short.</string>
+				</dict>
+			</array>
+			<key>type</key>
+			<string>sphinx</string>
+		</dict>
+	</array>
+	<key>files</key>
+	<array>
+		<string>files/sample.rst</string>
+	</array>
+	<key>metadata</key>
+	<dict>
+		<key>analyzer</key>
+		<dict>
+			<key>name</key>
+			<string>sphinx</string>
+		</dict>
+		<key>generated_by</key>
+		<dict>
+			<key>name</key>
+			<string>report-converter</string>
+			<key>version</key>
+			<string>x.y.z</string>
+		</dict>
+	</dict>
+</dict>
+</plist>

tools/report-converter/tests/unit/sphinx_output_test_files/sample.out

@@ -0,0 +1 @@
+/files/sample.rst:2: WARNING: Title underline too short.

tools/report-converter/tests/unit/test_sphinx_parser.py

@@ -0,0 +1,79 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+
+"""
+This module tests the correctness of the SphinxAnalyzerResult,
+which used in sequence transform Sphinx output to a plist file.
+"""
+
+
+import os
+import plistlib
+import shutil
+import tempfile
+import unittest
+
+
+from codechecker_report_converter.sphinx.analyzer_result import \
+    SphinxAnalyzerResult
+
+
+class SphinxResultTestCase(unittest.TestCase):
+    """ Test the output of the SphinxAnalyzerResult. """
+
+    def setUp(self):
+        """ Setup the test. """
+        self.analyzer_result = SphinxAnalyzerResult()
+        self.cc_result_dir = tempfile.mkdtemp()
+        self.test_files = os.path.join(os.path.dirname(__file__),
+                                       'sphinx_output_test_files')
+
+    def tearDown(self):
+        """ Clean temporary directory. """
+        shutil.rmtree(self.cc_result_dir)
+
+    def test_no_sphinx_output_file(self):
+        """ Test transforming single rst file. """
+        analyzer_result = os.path.join(self.test_files, 'files',
+                                       'sample.rst')
+
+        ret = self.analyzer_result.transform(analyzer_result,
+                                             self.cc_result_dir)
+        self.assertFalse(ret)
+
+    def test_transform_dir(self):
+        """ Test transforming a directory. """
+        analyzer_result = os.path.join(self.test_files)
+
+        ret = self.analyzer_result.transform(analyzer_result,
+                                             self.cc_result_dir)
+        self.assertFalse(ret)
+
+    def test_transform_single_file(self):
+        """ Test transforming single output file. """
+        analyzer_result = os.path.join(self.test_files, 'sample.out')
+        self.analyzer_result.transform(analyzer_result, self.cc_result_dir)
+
+        plist_file = os.path.join(self.cc_result_dir,
+                                  'sample.rst_sphinx.plist')
+
+        with open(plist_file, mode='rb') as pfile:
+            res = plistlib.load(pfile)
+
+            # Use relative path for this test.
+            res['files'][0] = os.path.join('files', 'sample.rst')
+
+            self.assertTrue(res['metadata']['generated_by']['version'])
+            res['metadata']['generated_by']['version'] = "x.y.z"
+
+        plist_file = os.path.join(self.test_files,
+                                  'sample.expected.plist')
+        with open(plist_file, mode='rb') as pfile:
+            exp = plistlib.load(pfile)
+
+        self.assertEqual(res, exp)