From 557e6b74fae89d58ac8343cbc934ee91c5edad76 Mon Sep 17 00:00:00 2001 From: Joshua Jay Herman Date: Thu, 27 Apr 2023 14:00:52 -0600 Subject: [PATCH 1/5] This allows the test runner to execute test_multiprocessing and test_asyncio in parallel. By sharding the individual tests in test_multiprocessing and test_asyncio we are able to reduce considerably the time it takes to run the whole test suite. --- Lib/test/libregrtest/runtest.py | 85 ++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e9bb72a7d77ee1..1df0dc18d70c8c 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -19,10 +19,10 @@ class TestResult: def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, ) -> None: self.name = name self.duration_sec = duration_sec @@ -39,12 +39,12 @@ def __str__(self) -> str: class Failed(TestResult): def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, - errors: list[tuple[str, str]] | None = None, - failures: list[tuple[str, str]] | None = None, + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, + errors: list[tuple[str, str]] | None = None, + failures: list[tuple[str, str]] | None = None, ) -> None: super().__init__(name, duration_sec=duration_sec, xml_data=xml_data) self.errors = errors @@ -128,21 +128,30 @@ def __str__(self) -> str: # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' ] # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +#If these test directories are encountered recurse into them and treat each +# test_ .py or dir as a separate test module. This can increase parallelism. +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. + +SPLITTESTDIRS = { + "test_asyncio", + "test_compiler", +} # Storage of uncollectable objects FOUND_GARBAGE = [] @@ -158,7 +167,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -166,8 +175,16 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) + if mod[:5] == "test_" and mod not in others: + if mod in splittestdirs: + subdir = os.path.join(testdir, mod) + if len(base_mod): + mod = f"{base_mod}.{mod}" + else: + mod = f"test.{mod}" + tests.extend(findtests(subdir, [], nottests, splittestdirs, mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod) return stdtests + sorted(tests) @@ -186,7 +203,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: output_on_failure = ns.verbose3 use_timeout = ( - ns.timeout is not None and threading_helper.can_start_thread + ns.timeout is not None and threading_helper.can_start_thread ) if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) @@ -217,7 +234,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: print_warning.orig_stderr = stream result = _runtest_inner(ns, test_name, - display_failure=False) + display_failure=False) if not isinstance(result, Passed): output = stream.getvalue() finally: @@ -233,13 +250,13 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: support.verbose = ns.verbose result = _runtest_inner(ns, test_name, - display_failure=not ns.verbose) + display_failure=not ns.verbose) if xml_list: import xml.etree.ElementTree as ET result.xml_data = [ - ET.tostring(x).decode('us-ascii') - for x in xml_list + ET.tostring(x).decode('us-ascii') + for x in xml_list ] result.duration_sec = time.perf_counter() - start_time @@ -267,7 +284,7 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) + file=sys.stderr, flush=True) return Failed(test_name) @@ -328,7 +345,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: if gc.garbage: support.environment_altered = True print_warning(f"{test_name} created {len(gc.garbage)} " - f"uncollectable object(s).") + f"uncollectable object(s).") # move the uncollectable objects somewhere, # so we don't see them again @@ -341,7 +358,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: def _runtest_inner( - ns: Namespace, test_name: str, display_failure: bool = True + ns: Namespace, test_name: str, display_failure: bool = True ) -> TestResult: # Detect environment changes, handle exceptions. @@ -387,7 +404,7 @@ def _runtest_inner( if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) + file=sys.stderr, flush=True) return UncaughtException(test_name) if refleak: @@ -415,7 +432,7 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None: kind, nuker = "file", os.unlink else: raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") + f"directory nor file") if verbose: print_warning(f"{test_name} left behind {kind} {name!r}") @@ -428,4 +445,4 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None: nuker(name) except Exception as exc: print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") + f"and it couldn't be removed: {exc}") From 345d12f94283d8e3430242ae52a964de646b188f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 28 Apr 2023 12:00:22 -0700 Subject: [PATCH 2/5] revert whitespace changes --- Lib/test/libregrtest/runtest.py | 66 ++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 1df0dc18d70c8c..31c83c4ba7a51d 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -19,10 +19,10 @@ class TestResult: def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, ) -> None: self.name = name self.duration_sec = duration_sec @@ -39,12 +39,12 @@ def __str__(self) -> str: class Failed(TestResult): def __init__( - self, - name: str, - duration_sec: float = 0.0, - xml_data: list[str] | None = None, - errors: list[tuple[str, str]] | None = None, - failures: list[tuple[str, str]] | None = None, + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, + errors: list[tuple[str, str]] | None = None, + failures: list[tuple[str, str]] | None = None, ) -> None: super().__init__(name, duration_sec=duration_sec, xml_data=xml_data) self.errors = errors @@ -128,16 +128,16 @@ def __str__(self) -> str: # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' ] # set of tests that we don't want to be executed when using regrtest @@ -149,8 +149,8 @@ def __str__(self) -> str: # __init__.py may do things which alter what tests are to be run. SPLITTESTDIRS = { - "test_asyncio", - "test_compiler", + "test_asyncio", + "test_compiler", } # Storage of uncollectable objects @@ -203,7 +203,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: output_on_failure = ns.verbose3 use_timeout = ( - ns.timeout is not None and threading_helper.can_start_thread + ns.timeout is not None and threading_helper.can_start_thread ) if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) @@ -234,7 +234,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: print_warning.orig_stderr = stream result = _runtest_inner(ns, test_name, - display_failure=False) + display_failure=False) if not isinstance(result, Passed): output = stream.getvalue() finally: @@ -250,13 +250,13 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: support.verbose = ns.verbose result = _runtest_inner(ns, test_name, - display_failure=not ns.verbose) + display_failure=not ns.verbose) if xml_list: import xml.etree.ElementTree as ET result.xml_data = [ - ET.tostring(x).decode('us-ascii') - for x in xml_list + ET.tostring(x).decode('us-ascii') + for x in xml_list ] result.duration_sec = time.perf_counter() - start_time @@ -284,7 +284,7 @@ def runtest(ns: Namespace, test_name: str) -> TestResult: if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) + file=sys.stderr, flush=True) return Failed(test_name) @@ -345,7 +345,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: if gc.garbage: support.environment_altered = True print_warning(f"{test_name} created {len(gc.garbage)} " - f"uncollectable object(s).") + f"uncollectable object(s).") # move the uncollectable objects somewhere, # so we don't see them again @@ -358,7 +358,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool: def _runtest_inner( - ns: Namespace, test_name: str, display_failure: bool = True + ns: Namespace, test_name: str, display_failure: bool = True ) -> TestResult: # Detect environment changes, handle exceptions. @@ -404,7 +404,7 @@ def _runtest_inner( if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) + file=sys.stderr, flush=True) return UncaughtException(test_name) if refleak: @@ -432,7 +432,7 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None: kind, nuker = "file", os.unlink else: raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") + f"directory nor file") if verbose: print_warning(f"{test_name} left behind {kind} {name!r}") @@ -445,4 +445,4 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None: nuker(name) except Exception as exc: print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") + f"and it couldn't be removed: {exc}") From 9b71c206b7d27703f83f02c79e928f97e610b0c9 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 28 Apr 2023 12:02:40 -0700 Subject: [PATCH 3/5] multiprocessing, not compiler --- Lib/test/libregrtest/runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 31c83c4ba7a51d..1efd21abb13a9a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -150,7 +150,7 @@ def __str__(self) -> str: SPLITTESTDIRS = { "test_asyncio", - "test_compiler", + "test_multiprocessing", } # Storage of uncollectable objects From 19c6c73959eb9124440fa17eb614cf20d3105fb9 Mon Sep 17 00:00:00 2001 From: Joshua Jay Herman Date: Sat, 29 Apr 2023 08:37:32 -0500 Subject: [PATCH 4/5] This removes an extraneous test package that doesn't exist and also code cleanup. For the new test arguments they were made keyword only and also test_multiprocessing was removed from the SPLITTESTDIRS. The argument splittestdirs was renamed to split_test_dirs for findtests. --- Lib/test/libregrtest/runtest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 1efd21abb13a9a..db6668ef396d96 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -150,7 +150,6 @@ def __str__(self) -> str: SPLITTESTDIRS = { "test_asyncio", - "test_multiprocessing", } # Storage of uncollectable objects @@ -167,7 +166,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs=SPLITTESTDIRS, base_mod=""): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -176,13 +175,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs= for name in names: mod, ext = os.path.splitext(name) if mod[:5] == "test_" and mod not in others: - if mod in splittestdirs: + if mod in split_test_dirs: subdir = os.path.join(testdir, mod) if len(base_mod): mod = f"{base_mod}.{mod}" else: mod = f"test.{mod}" - tests.extend(findtests(subdir, [], nottests, splittestdirs, mod)) + tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) elif ext in (".py", ""): tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod) return stdtests + sorted(tests) From b409e2cf4ae6323fd9de88b00f265ae5884e64ba Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 29 Apr 2023 16:40:25 -0700 Subject: [PATCH 5/5] simplify logic: no len()s, tight f-expression. --- Lib/test/libregrtest/runtest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index db6668ef396d96..61595277ed6d5a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -177,13 +177,10 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_ if mod[:5] == "test_" and mod not in others: if mod in split_test_dirs: subdir = os.path.join(testdir, mod) - if len(base_mod): - mod = f"{base_mod}.{mod}" - else: - mod = f"test.{mod}" + mod = f"{base_mod or 'test'}.{mod}" tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) elif ext in (".py", ""): - tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod) + tests.append(f"{base_mod}.{mod}" if base_mod else mod) return stdtests + sorted(tests)