pytest: add scripts/check_pytest.py checking if tests are listed

On high level, this simply lists all pytest/tests/**/*.py files and
checks whether they are listed somewhere in nightly/*.txt files.  The
end goal is to have this as a required check before a commit is
accepted so that no pytest’s are forgotten.

Currently though, there are 10 files which the script complains
about.  The output of the script is:

    Found 9 pytest files (i.e. Python files in pytest/tests directory)
    which are not included in any of the nightly/*.txt files:

     -  pytest/tests/adversarial/gc_rollback.py
     -  pytest/tests/sanity/concurrent_function_calls.py
     -  pytest/tests/sanity/proxy_example.py
     -  pytest/tests/sanity/restart.py
     -  pytest/tests/sanity/rpc_finality.py
     -  pytest/tests/sanity/rpc_tx_submission.py
     -  pytest/tests/sanity/state_migration.py
     -  pytest/tests/stress/network_stress.py
     -  pytest/tests/stress/saturate_routing_table.py

    Add the tests to one of the lists in nightly/*.txt files.  For example
    as:

        pytest sanity/rpc_tx_submission.py

    If the test is temporarily disabled, add it to one of the files with an
    appropriate TODO comment.  For example:

        # TODO(near#1234): Enable the test again once <some condition>
        # pytest sanity/rpc_tx_submission.py

    Note that the TODO comment must reference a GitHub issue (i.e. must
    contain a #<number> string).

    If the file is not a test but a helper library, consider moving it out
    of the pytest/tests directory to pytest/lib or add it to HELPER_SCRIPTS
    list at the top of scripts/check_pytests.py file.

Before this check is incorporated in checks pipeline, the above needs
to be resolved.

Issue: near#2736

scripts/check_pytests.py

@@ -0,0 +1,189 @@
+"""Checks whether all pytest tests are mentioned in NayDuck or Buildkite
+
+Lists all Python scripts inside of pytest/tests directory and checks whether
+they are referenced in NayDuck test list files (the nightly/*.txt files) or
+Buildkite pipeline configuration (the .buildkite/pipeline.yml file).  Returns
+with success if that's the case; with failure otherwise.
+"""
+import fnmatch
+import os
+import pathlib
+import random
+import re
+import sys
+import typing
+
+import yaml
+
+# List of globs of Python scripts in the pytest/tests directory which are not
+# test but rather helper scripts and libraries.
+HELPER_SCRIPTS = [
+    'companion.py',
+    'delete_remote_nodes.py',
+    'mocknet/load_testing_helper.py',
+    'stress/hundred_nodes/*',
+    'tests/delete_remote_nodes.py',
+]
+
+
+PYTEST_TESTS_DIRECTORY = pathlib.Path('pytest/tests')
+NIGHTLY_DIRECTORY = pathlib.Path('nightly')
+BUILDKITE_PIPELINE_FILE = pathlib.Path('.buildkite/pipeline.yml')
+
+StrGenerator = typing.Generator[str, None, None]
+
+
+def list_test_files(topdir: pathlib.Path) -> StrGenerator:
+    """Yields all *.py files in a given directory traversing it recursively.
+
+    Args:
+        topdir: Path to the directory to traverse.  Directory is traversed
+            recursively.
+    Yields:
+        Paths (as str objects) to all the Python source files in the directory
+        relative to the top directory.  __init__.py files (and in fact any files
+        starting with __) are ignored.
+    """
+    for dirname, _, filenames in os.walk(topdir):
+        dirpath = pathlib.Path(dirname).relative_to(topdir)
+        for filename in filenames:
+            if not filename.startswith('__') and filename.endswith('.py'):
+                yield str(dirpath / filename)
+
+
+def list_nayduck_tests(directory: pathlib.Path) -> StrGenerator:
+    """Reads all NayDuck test list files and yields all tests mentioned there.
+
+    The NayDuck test list files are ones with .txt extension.  Only pytest and
+    mocknet tests are taken into account and returned.  Enabled tests as well as
+    those commented out with a corresponding TODO comment are considered.
+
+    Args:
+        directory: Path to the directory where to look for *.txt files.
+            Directory is not traversed recursively; i.e. only files directly in
+            the directory are read.
+    Yields:
+        pytest and mocknet tests mentioned in the test list files.  May include
+        duplicates.
+    """
+    for filename in os.listdir(directory):
+        if filename.endswith('.txt'):
+            with open(directory / filename) as rd:
+                yield from read_nayduck_tests(rd)
+
+
+def read_nayduck_tests(rd: typing.TextIO) -> StrGenerator:
+    """Reads NayDuck test file and yields all tests mentioned there.
+
+    Args:
+        rd: An open text stream for the NayDuck test list file.
+    Yields:
+        pytest and mocknet tests mentioned in the file.  May include duplicates.
+    """
+    def extract_name(line: str) -> StrGenerator:
+        tokens = line.split()
+        try:
+            idx = 1 + (tokens[0] == '#')
+            idx += tokens[idx].startswith('--timeout')
+            yield tokens[idx]
+        except ValueError:
+            pass
+
+    found_todo = False
+    for line in rd:
+        line = line.strip()
+        line = re.sub(r'\s+', ' ', line)
+        if re.search(r'^\s*(?:pytest|mocknet)\s+', line):
+            found_todo = False
+            yield from extract_name(line)
+        elif found_todo and re.search(r'^\s*#\s*(?:pytest|mocknet)\s+', line):
+            yield from extract_name(line)
+        elif re.search(r'^\s*#\s*TO' r'DO.*#[0-9]{4,}', line):
+            found_todo = True
+        elif not line.strip().startswith('#'):
+            found_todo = False
+
+
+def read_pipeline_tests(filename: pathlib.Path) -> StrGenerator:
+    """Reads pytest tests mentioned in Buildkite pipeline file.
+
+    The parsing of the pipeline configuration is quite naive.  All this function
+    is looking for is a "cd pytest" line in a step's command followed by
+    "python3 tests/<name>" lines.  The <name> is yielded for each such line.
+
+    Args:
+        filename: Path to the Buildkite pipeline configuration file.
+    Yields:
+        pytest tests mentioned in the commands in the configuration file.
+    """
+    with open(filename) as rd:
+        data = yaml.load(rd, Loader=yaml.SafeLoader)
+    for step in data.get('steps', ()):
+        in_pytest = False
+        for line in step.get('command', '').splitlines():
+            line = line.strip()
+            line = re.sub(r'\s+', ' ', line)
+            if line == 'cd pytest':
+                in_pytest = True
+            elif in_pytest and line.startswith('python3 tests/'):
+                yield line.split()[1][6:]
+
+
+def print_error(missing: typing.Collection[str]) -> None:
+    """Formats and outputs an error message listing missing tests."""
+    this_file = os.path.relpath(__file__)
+    example = random.sample(tuple(missing), 1)[0]
+    if example.startswith('mocknet/'):
+        example = 'mocknet ' + example
+    else:
+        example = 'pytest ' + example
+    msg = '''\
+Found {count} pytest file{s} (i.e. Python file{s} in pytest/tests directory)
+which are not included in any of the nightly/*.txt files:
+
+{missing}
+
+Add the test{s} to one of the lists in nightly/*.txt files.  For example as:
+
+    {example}
+
+If the test is temporarily disabled, add it to one of the files with an
+appropriate {todo} comment.  For example:
+
+    # {todo}(#1234): Enable the test again once <some condition>
+    # {example}
+
+Note that the {todo} comment must reference a GitHub issue (i.e. must
+contain a #<number> string).
+
+If the file is not a test but a helper library, consider moving it out
+of the pytest/tests directory to pytest/lib or add it to HELPER_SCRIPTS
+list at the top of {this_file} file.'''.format(
+    count=len(missing),
+    s='' if len(missing) == 1 else 's',
+    missing='\n'.join(' -  pytest/tests/' + name for name in sorted(missing)),
+    example=example,
+    this_file=this_file,
+    todo='TO' 'DO',
+)
+    print(msg, file=sys.stderr)
+
+
+def main() -> int:
+    """Main function of the script; returns integer exit code."""
+    missing = set(list_test_files(PYTEST_TESTS_DIRECTORY))
+    missing.difference_update(list_nayduck_tests(NIGHTLY_DIRECTORY))
+    missing.difference_update(read_pipeline_tests(BUILDKITE_PIPELINE_FILE))
+    missing = set(filename
+                  for filename in missing
+                  if not any(fnmatch.fnmatch(filename, pattern)
+                             for pattern in HELPER_SCRIPTS))
+    if missing:
+        print_error(missing)
+        return 1
+    print('All tests included')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())