From 251802607688b47b91c1ccca54a715bd07d261a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:16:00 +0100 Subject: [PATCH] Add CI test for message documentation (#5956) --- .github/workflows/ci.yaml | 31 ++++++ doc/data/messages/e/empty-docstring/bad.py | 4 +- doc/test_messages_documentation.py | 110 +++++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 doc/test_messages_documentation.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5bf14f0601..8432c93fbe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -148,6 +148,37 @@ jobs: . venv/bin/activate pytest tests/ -k unittest_spelling + documentation: + name: checks / documentation + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: prepare-base + steps: + - name: Check out code from GitHub + uses: actions/checkout@v3.0.0 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + id: python + uses: actions/setup-python@v3.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v3.0.0 + with: + path: venv + key: + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.prepare-base.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python venv from cache" + exit 1 + - name: Run checks on documentation code examples + run: | + . venv/bin/activate + pytest doc/test_messages_documentation.py + prepare-tests-linux: name: tests / prepare / ${{ matrix.python-version }} / Linux runs-on: ubuntu-latest diff --git a/doc/data/messages/e/empty-docstring/bad.py b/doc/data/messages/e/empty-docstring/bad.py index 83d3601251..33dbb5f881 100644 --- a/doc/data/messages/e/empty-docstring/bad.py +++ b/doc/data/messages/e/empty-docstring/bad.py @@ -1,2 +1,2 @@ -def foo(): - pass # [emtpy-docstring] +def foo(): # [empty-docstring] + """""" diff --git a/doc/test_messages_documentation.py b/doc/test_messages_documentation.py new file mode 100644 index 0000000000..06c71d725c --- /dev/null +++ b/doc/test_messages_documentation.py @@ -0,0 +1,110 @@ +"""Functional tests for the code examples in the messages documentation.""" + +from collections import Counter +from pathlib import Path +from typing import Counter as CounterType +from typing import List, TextIO, Tuple + +import pytest + +from pylint import checkers +from pylint.config.config_initialization import _config_initialization +from pylint.lint import PyLinter +from pylint.message.message import Message +from pylint.testutils.constants import _EXPECTED_RE +from pylint.testutils.reporter_for_tests import FunctionalTestReporter + +MessageCounter = CounterType[Tuple[int, str]] + + +def get_functional_test_files_from_directory(input_dir: Path) -> List[Tuple[str, Path]]: + """Get all functional tests in the input_dir.""" + suite: List[Tuple[str, Path]] = [] + + for subdirectory in input_dir.iterdir(): + for message_dir in subdirectory.iterdir(): + if (message_dir / "good.py").exists(): + suite.append( + (message_dir.stem, message_dir / "good.py"), + ) + if (message_dir / "bad.py").exists(): + suite.append( + (message_dir.stem, message_dir / "bad.py"), + ) + return suite + + +TESTS_DIR = Path(__file__).parent.resolve() / "data" / "messages" +TESTS = get_functional_test_files_from_directory(TESTS_DIR) +TESTS_NAMES = [f"{t[0]}-{t[1].stem}" for t in TESTS] + + +class LintModuleTest: + def __init__(self, test_file: Tuple[str, Path]) -> None: + self._test_file = test_file + + _test_reporter = FunctionalTestReporter() + + self._linter = PyLinter() + self._linter.config.persistent = 0 + checkers.initialize(self._linter) + + _config_initialization( + self._linter, + args_list=[ + str(test_file[1]), + "--disable=all", + f"--enable={test_file[0]}", + ], + reporter=_test_reporter, + ) + + def runTest(self) -> None: + self._runTest() + + @staticmethod + def get_expected_messages(stream: TextIO) -> MessageCounter: + """Parse a file and get expected messages.""" + messages: MessageCounter = Counter() + for i, line in enumerate(stream): + match = _EXPECTED_RE.search(line) + if match is None: + continue + + line = match.group("line") + if line is None: + lineno = i + 1 + else: + lineno = int(line) + + for msg_id in match.group("msgs").split(","): + messages[lineno, msg_id.strip()] += 1 + return messages + + def _get_expected(self) -> MessageCounter: + """Get the expected messages for a file.""" + with open(self._test_file[1], encoding="utf8") as f: + expected_msgs = self.get_expected_messages(f) + return expected_msgs + + def _get_actual(self) -> MessageCounter: + """Get the actual messages after a run.""" + messages: List[Message] = self._linter.reporter.messages + messages.sort(key=lambda m: (m.line, m.symbol, m.msg)) + received_msgs: MessageCounter = Counter() + for msg in messages: + received_msgs[msg.line, msg.symbol] += 1 + return received_msgs + + def _runTest(self) -> None: + """Run the test and assert message differences.""" + self._linter.check([str(self._test_file[1])]) + expected_messages = self._get_expected() + actual_messages = self._get_actual() + assert expected_messages == actual_messages + + +@pytest.mark.parametrize("test_file", TESTS, ids=TESTS_NAMES) +def test_code_examples(test_file: Tuple[str, Path]) -> None: + lint_test = LintModuleTest(test_file) + lint_test.runTest()