From ee0ac328d38a86f7907598c94cb88a97635b32f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Jan 2022 18:46:09 +0100 Subject: [PATCH] bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bpo-46542: test_lib2to3 uses support.infinite_recursion() Fix a Python crash in test_lib2to3 when using Python built in debug mode: limit the recursion limit. The test_all_project_files() test of test_lib2to3 now uses the test.support.infinite_recursion() context manager when processing the infinite_recursion.py file to prevent a crash when Python is built in debug mode. The two test_all_project_files() tests now use subTest() and log the refactored/parsed filename (if test_lib2to3 is run in verbose mode). * Update Lib/lib2to3/tests/data/infinite_recursion.py Co-authored-by: Jelle Zijlstra Co-authored-by: Łukasz Langa Co-authored-by: Jelle Zijlstra --- Lib/lib2to3/pytree.py | 4 +- Lib/lib2to3/tests/data/infinite_recursion.py | 4 +- Lib/lib2to3/tests/test_all_fixers.py | 19 ++++++++- Lib/lib2to3/tests/test_parser.py | 41 +++++++++++-------- .../2022-01-31-17-34-13.bpo-46542.RTMm1T.rst | 2 + 5 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py index 2a6ef2ef5240a2..729023df0284e1 100644 --- a/Lib/lib2to3/pytree.py +++ b/Lib/lib2to3/pytree.py @@ -720,8 +720,8 @@ def generate_matches(self, nodes): r[self.name] = nodes[:count] yield count, r except RuntimeError: - # We fall back to the iterative pattern matching scheme if the recursive - # scheme hits the recursion limit. + # Fall back to the iterative pattern matching scheme if the + # recursive scheme hits the recursion limit (RecursionError). for count, r in self._iterative_matches(nodes): if self.name: r[self.name] = nodes[:count] diff --git a/Lib/lib2to3/tests/data/infinite_recursion.py b/Lib/lib2to3/tests/data/infinite_recursion.py index 71715ef29dc488..acc62ed4f1a44c 100644 --- a/Lib/lib2to3/tests/data/infinite_recursion.py +++ b/Lib/lib2to3/tests/data/infinite_recursion.py @@ -1,5 +1,5 @@ -# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching -# scheme in the event that the faster recursive system fails due to infinite recursion. +# Verify that 2to3 falls back from the recursive pattern matching scheme to a +# slower, iterative scheme in the event of a RecursionError. from ctypes import * STRING = c_char_p diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py index c0507cf3fbcbfe..a265941490d5dc 100644 --- a/Lib/lib2to3/tests/test_all_fixers.py +++ b/Lib/lib2to3/tests/test_all_fixers.py @@ -6,8 +6,10 @@ # Author: Collin Winter # Python imports -import unittest +import os.path +import sys import test.support +import unittest # Local imports from . import support @@ -19,9 +21,22 @@ class Test_all(support.TestCase): def setUp(self): self.refactor = support.get_refactorer() + def refactor_file(self, filepath): + if test.support.verbose: + print(f"Refactor file: {filepath}") + if os.path.basename(filepath) == 'infinite_recursion.py': + # bpo-46542: Processing infinite_recursion.py can crash Python + # if Python is built in debug mode: lower the recursion limit + # to prevent a crash. + with test.support.infinite_recursion(150): + self.refactor.refactor_file(filepath) + else: + self.refactor.refactor_file(filepath) + def test_all_project_files(self): for filepath in support.all_project_files(): - self.refactor.refactor_file(filepath) + with self.subTest(filepath=filepath): + self.refactor_file(filepath) if __name__ == '__main__': unittest.main() diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index ff4f8078878d8d..74a57875742092 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -20,6 +20,7 @@ import subprocess import sys import tempfile +import test.support import unittest # Local imports @@ -589,25 +590,31 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" + def parse_file(self, filepath): + if test.support.verbose: + print(f"Parse file: {filepath}") + with open(filepath, "rb") as fp: + encoding = tokenize.detect_encoding(fp.readline)[0] + self.assertIsNotNone(encoding, + "can't detect encoding for %s" % filepath) + with open(filepath, "r", encoding=encoding) as fp: + source = fp.read() + try: + tree = driver.parse_string(source) + except ParseError: + try: + tree = driver_no_print_statement.parse_string(source) + except ParseError as err: + self.fail('ParseError on file %s (%s)' % (filepath, err)) + new = str(tree) + if new != source: + print(diff_texts(source, new, filepath)) + self.fail("Idempotency failed: %s" % filepath) + def test_all_project_files(self): for filepath in support.all_project_files(): - with open(filepath, "rb") as fp: - encoding = tokenize.detect_encoding(fp.readline)[0] - self.assertIsNotNone(encoding, - "can't detect encoding for %s" % filepath) - with open(filepath, "r", encoding=encoding) as fp: - source = fp.read() - try: - tree = driver.parse_string(source) - except ParseError: - try: - tree = driver_no_print_statement.parse_string(source) - except ParseError as err: - self.fail('ParseError on file %s (%s)' % (filepath, err)) - new = str(tree) - if new != source: - print(diff_texts(source, new, filepath)) - self.fail("Idempotency failed: %s" % filepath) + with self.subTest(filepath=filepath): + self.parse_file(filepath) def test_extended_unpacking(self): driver.parse_string("a, *b, c = x\n") diff --git a/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst new file mode 100644 index 00000000000000..5596498724930f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst @@ -0,0 +1,2 @@ +Fix a Python crash in test_lib2to3 when using Python built in debug mode: +limit the recursion limit. Patch by Victor Stinner.