Skip to content

Commit

Permalink
bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035)
Browse files Browse the repository at this point in the history
* bpo-46542: test_lib2to3 uses support.infinite_recursion()

Fix a Python crash in test_lib2to3 when using Python built in debug
mode: limit the recursion limit.

The test_all_project_files() test of test_lib2to3 now uses the
test.support.infinite_recursion() context manager when processing the
infinite_recursion.py file to prevent a crash when Python is built in
debug mode.

The two test_all_project_files() tests now use subTest() and log the
refactored/parsed filename (if test_lib2to3 is run in verbose mode).

* Update Lib/lib2to3/tests/data/infinite_recursion.py

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

Co-authored-by: Łukasz Langa <lukasz@langa.pl>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
  • Loading branch information
3 people authored Jan 31, 2022
1 parent 7685693 commit ee0ac32
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 23 deletions.
4 changes: 2 additions & 2 deletions Lib/lib2to3/pytree.py
Original file line number Diff line number Diff line change
Expand Up @@ -720,8 +720,8 @@ def generate_matches(self, nodes):
r[self.name] = nodes[:count]
yield count, r
except RuntimeError:
# We fall back to the iterative pattern matching scheme if the recursive
# scheme hits the recursion limit.
# Fall back to the iterative pattern matching scheme if the
# recursive scheme hits the recursion limit (RecursionError).
for count, r in self._iterative_matches(nodes):
if self.name:
r[self.name] = nodes[:count]
Expand Down
4 changes: 2 additions & 2 deletions Lib/lib2to3/tests/data/infinite_recursion.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching
# scheme in the event that the faster recursive system fails due to infinite recursion.
# Verify that 2to3 falls back from the recursive pattern matching scheme to a
# slower, iterative scheme in the event of a RecursionError.
from ctypes import *
STRING = c_char_p

Expand Down
19 changes: 17 additions & 2 deletions Lib/lib2to3/tests/test_all_fixers.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
# Author: Collin Winter

# Python imports
import unittest
import os.path
import sys
import test.support
import unittest

# Local imports
from . import support
Expand All @@ -19,9 +21,22 @@ class Test_all(support.TestCase):
def setUp(self):
self.refactor = support.get_refactorer()

def refactor_file(self, filepath):
if test.support.verbose:
print(f"Refactor file: {filepath}")
if os.path.basename(filepath) == 'infinite_recursion.py':
# bpo-46542: Processing infinite_recursion.py can crash Python
# if Python is built in debug mode: lower the recursion limit
# to prevent a crash.
with test.support.infinite_recursion(150):
self.refactor.refactor_file(filepath)
else:
self.refactor.refactor_file(filepath)

def test_all_project_files(self):
for filepath in support.all_project_files():
self.refactor.refactor_file(filepath)
with self.subTest(filepath=filepath):
self.refactor_file(filepath)

if __name__ == '__main__':
unittest.main()
41 changes: 24 additions & 17 deletions Lib/lib2to3/tests/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import subprocess
import sys
import tempfile
import test.support
import unittest

# Local imports
Expand Down Expand Up @@ -589,25 +590,31 @@ class TestParserIdempotency(support.TestCase):

"""A cut-down version of pytree_idempotency.py."""

def parse_file(self, filepath):
if test.support.verbose:
print(f"Parse file: {filepath}")
with open(filepath, "rb") as fp:
encoding = tokenize.detect_encoding(fp.readline)[0]
self.assertIsNotNone(encoding,
"can't detect encoding for %s" % filepath)
with open(filepath, "r", encoding=encoding) as fp:
source = fp.read()
try:
tree = driver.parse_string(source)
except ParseError:
try:
tree = driver_no_print_statement.parse_string(source)
except ParseError as err:
self.fail('ParseError on file %s (%s)' % (filepath, err))
new = str(tree)
if new != source:
print(diff_texts(source, new, filepath))
self.fail("Idempotency failed: %s" % filepath)

def test_all_project_files(self):
for filepath in support.all_project_files():
with open(filepath, "rb") as fp:
encoding = tokenize.detect_encoding(fp.readline)[0]
self.assertIsNotNone(encoding,
"can't detect encoding for %s" % filepath)
with open(filepath, "r", encoding=encoding) as fp:
source = fp.read()
try:
tree = driver.parse_string(source)
except ParseError:
try:
tree = driver_no_print_statement.parse_string(source)
except ParseError as err:
self.fail('ParseError on file %s (%s)' % (filepath, err))
new = str(tree)
if new != source:
print(diff_texts(source, new, filepath))
self.fail("Idempotency failed: %s" % filepath)
with self.subTest(filepath=filepath):
self.parse_file(filepath)

def test_extended_unpacking(self):
driver.parse_string("a, *b, c = x\n")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix a Python crash in test_lib2to3 when using Python built in debug mode:
limit the recursion limit. Patch by Victor Stinner.

0 comments on commit ee0ac32

Please sign in to comment.