From ecb5fa299181e1516b616455237398d0ecb2b2cb Mon Sep 17 00:00:00 2001 From: Michal Nazarewicz Date: Sun, 18 Jul 2021 04:23:17 +0200 Subject: [PATCH] pytest: add scripts/check_pytest.py checking if tests are listed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On high level, this simply lists all pytest/tests/**/*.py files and checks whether they are listed somewhere in nightly/*.txt files. The end goal is to have this as a required check before a commit is accepted so that no pytest’s are forgotten. Currently though, there are 10 files which the script complains about. The output of the script is: Found 9 pytest files (i.e. Python files in pytest/tests directory) which are not included in any of the nightly/*.txt files: - pytest/tests/adversarial/gc_rollback.py - pytest/tests/sanity/concurrent_function_calls.py - pytest/tests/sanity/proxy_example.py - pytest/tests/sanity/restart.py - pytest/tests/sanity/rpc_finality.py - pytest/tests/sanity/rpc_tx_submission.py - pytest/tests/sanity/state_migration.py - pytest/tests/stress/network_stress.py - pytest/tests/stress/saturate_routing_table.py Add the tests to one of the lists in nightly/*.txt files. For example as: pytest sanity/rpc_tx_submission.py If the test is temporarily disabled, add it to one of the files with an appropriate TODO comment. For example: # TODO(#1234): Enable the test again once # pytest sanity/rpc_tx_submission.py Note that the TODO comment must reference a GitHub issue (i.e. must contain a # string). If the file is not a test but a helper library, consider moving it out of the pytest/tests directory to pytest/lib or add it to HELPER_SCRIPTS list at the top of scripts/check_pytests.py file. Before this check is incorporated in checks pipeline, the above needs to be resolved. Issue: https://github.com/near/nearcore/issues/2736 --- scripts/check_pytests.py | 189 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 scripts/check_pytests.py diff --git a/scripts/check_pytests.py b/scripts/check_pytests.py new file mode 100644 index 00000000000..011031cc962 --- /dev/null +++ b/scripts/check_pytests.py @@ -0,0 +1,189 @@ +"""Checks whether all pytest tests are mentioned in NayDuck or Buildkite + +Lists all Python scripts inside of pytest/tests directory and checks whether +they are referenced in NayDuck test list files (the nightly/*.txt files) or +Buildkite pipeline configuration (the .buildkite/pipeline.yml file). Returns +with success if that's the case; with failure otherwise. +""" +import fnmatch +import os +import pathlib +import random +import re +import sys +import typing + +import yaml + +# List of globs of Python scripts in the pytest/tests directory which are not +# test but rather helper scripts and libraries. +HELPER_SCRIPTS = [ + 'companion.py', + 'delete_remote_nodes.py', + 'mocknet/load_testing_helper.py', + 'stress/hundred_nodes/*', + 'tests/delete_remote_nodes.py', +] + + +PYTEST_TESTS_DIRECTORY = pathlib.Path('pytest/tests') +NIGHTLY_DIRECTORY = pathlib.Path('nightly') +BUILDKITE_PIPELINE_FILE = pathlib.Path('.buildkite/pipeline.yml') + +StrGenerator = typing.Generator[str, None, None] + + +def list_test_files(topdir: pathlib.Path) -> StrGenerator: + """Yields all *.py files in a given directory traversing it recursively. + + Args: + topdir: Path to the directory to traverse. Directory is traversed + recursively. + Yields: + Paths (as str objects) to all the Python source files in the directory + relative to the top directory. __init__.py files (and in fact any files + starting with __) are ignored. + """ + for dirname, _, filenames in os.walk(topdir): + dirpath = pathlib.Path(dirname).relative_to(topdir) + for filename in filenames: + if not filename.startswith('__') and filename.endswith('.py'): + yield str(dirpath / filename) + + +def list_nayduck_tests(directory: pathlib.Path) -> StrGenerator: + """Reads all NayDuck test list files and yields all tests mentioned there. + + The NayDuck test list files are ones with .txt extension. Only pytest and + mocknet tests are taken into account and returned. Enabled tests as well as + those commented out with a corresponding TODO comment are considered. + + Args: + directory: Path to the directory where to look for *.txt files. + Directory is not traversed recursively; i.e. only files directly in + the directory are read. + Yields: + pytest and mocknet tests mentioned in the test list files. May include + duplicates. + """ + for filename in os.listdir(directory): + if filename.endswith('.txt'): + with open(directory / filename) as rd: + yield from read_nayduck_tests(rd) + + +def read_nayduck_tests(rd: typing.TextIO) -> StrGenerator: + """Reads NayDuck test file and yields all tests mentioned there. + + Args: + rd: An open text stream for the NayDuck test list file. + Yields: + pytest and mocknet tests mentioned in the file. May include duplicates. + """ + def extract_name(line: str) -> StrGenerator: + tokens = line.split() + try: + idx = 1 + (tokens[0] == '#') + idx += tokens[idx].startswith('--timeout') + yield tokens[idx] + except ValueError: + pass + + found_todo = False + for line in rd: + line = line.strip() + line = re.sub(r'\s+', ' ', line) + if re.search(r'^\s*(?:pytest|mocknet)\s+', line): + found_todo = False + yield from extract_name(line) + elif found_todo and re.search(r'^\s*#\s*(?:pytest|mocknet)\s+', line): + yield from extract_name(line) + elif re.search(r'^\s*#\s*TO' r'DO.*#[0-9]{4,}', line): + found_todo = True + elif not line.strip().startswith('#'): + found_todo = False + + +def read_pipeline_tests(filename: pathlib.Path) -> StrGenerator: + """Reads pytest tests mentioned in Buildkite pipeline file. + + The parsing of the pipeline configuration is quite naive. All this function + is looking for is a "cd pytest" line in a step's command followed by + "python3 tests/" lines. The is yielded for each such line. + + Args: + filename: Path to the Buildkite pipeline configuration file. + Yields: + pytest tests mentioned in the commands in the configuration file. + """ + with open(filename) as rd: + data = yaml.load(rd, Loader=yaml.SafeLoader) + for step in data.get('steps', ()): + in_pytest = False + for line in step.get('command', '').splitlines(): + line = line.strip() + line = re.sub(r'\s+', ' ', line) + if line == 'cd pytest': + in_pytest = True + elif in_pytest and line.startswith('python3 tests/'): + yield line.split()[1][6:] + + +def print_error(missing: typing.Collection[str]) -> None: + """Formats and outputs an error message listing missing tests.""" + this_file = os.path.relpath(__file__) + example = random.sample(tuple(missing), 1)[0] + if example.startswith('mocknet/'): + example = 'mocknet ' + example + else: + example = 'pytest ' + example + msg = '''\ +Found {count} pytest file{s} (i.e. Python file{s} in pytest/tests directory) +which are not included in any of the nightly/*.txt files: + +{missing} + +Add the test{s} to one of the lists in nightly/*.txt files. For example as: + + {example} + +If the test is temporarily disabled, add it to one of the files with an +appropriate {todo} comment. For example: + + # {todo}(#1234): Enable the test again once + # {example} + +Note that the {todo} comment must reference a GitHub issue (i.e. must +contain a # string). + +If the file is not a test but a helper library, consider moving it out +of the pytest/tests directory to pytest/lib or add it to HELPER_SCRIPTS +list at the top of {this_file} file.'''.format( + count=len(missing), + s='' if len(missing) == 1 else 's', + missing='\n'.join(' - pytest/tests/' + name for name in sorted(missing)), + example=example, + this_file=this_file, + todo='TO' 'DO', +) + print(msg, file=sys.stderr) + + +def main() -> int: + """Main function of the script; returns integer exit code.""" + missing = set(list_test_files(PYTEST_TESTS_DIRECTORY)) + missing.difference_update(list_nayduck_tests(NIGHTLY_DIRECTORY)) + missing.difference_update(read_pipeline_tests(BUILDKITE_PIPELINE_FILE)) + missing = set(filename + for filename in missing + if not any(fnmatch.fnmatch(filename, pattern) + for pattern in HELPER_SCRIPTS)) + if missing: + print_error(missing) + return 1 + print('All tests included') + return 0 + + +if __name__ == '__main__': + sys.exit(main())