https://github.com/python/cpython/commit/30463860796790d3be97b76744290e4c364... commit: 30463860796790d3be97b76744290e4c364ad765 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: miss-islington <31488909+miss-islington@users.noreply.github.com> date: 2022-01-31T12:03:44-08:00 summary: bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035) * bpo-46542: test_lib2to3 uses support.infinite_recursion() Fix a Python crash in test_lib2to3 when using Python built in debug mode: limit the recursion limit. The test_all_project_files() test of test_lib2to3 now uses the test.support.infinite_recursion() context manager when processing the infinite_recursion.py file to prevent a crash when Python is built in debug mode. The two test_all_project_files() tests now use subTest() and log the refactored/parsed filename (if test_lib2to3 is run in verbose mode). * Update Lib/lib2to3/tests/data/infinite_recursion.py Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> Co-authored-by: Łukasz Langa <lukasz@langa.pl> Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> (cherry picked from commit ee0ac328d38a86f7907598c94cb88a97635b32f8) Co-authored-by: Victor Stinner <vstinner@python.org> files: A Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst M Lib/lib2to3/pytree.py M Lib/lib2to3/tests/data/infinite_recursion.py M Lib/lib2to3/tests/test_all_fixers.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py index 2a6ef2ef5240a..729023df0284e 100644 --- a/Lib/lib2to3/pytree.py +++ b/Lib/lib2to3/pytree.py @@ -720,8 +720,8 @@ def generate_matches(self, nodes): r[self.name] = nodes[:count] yield count, r except RuntimeError: - # We fall back to the iterative pattern matching scheme if the recursive - # scheme hits the recursion limit. + # Fall back to the iterative pattern matching scheme if the + # recursive scheme hits the recursion limit (RecursionError). for count, r in self._iterative_matches(nodes): if self.name: r[self.name] = nodes[:count] diff --git a/Lib/lib2to3/tests/data/infinite_recursion.py b/Lib/lib2to3/tests/data/infinite_recursion.py index 71715ef29dc48..acc62ed4f1a44 100644 --- a/Lib/lib2to3/tests/data/infinite_recursion.py +++ b/Lib/lib2to3/tests/data/infinite_recursion.py @@ -1,5 +1,5 @@ -# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching -# scheme in the event that the faster recursive system fails due to infinite recursion. +# Verify that 2to3 falls back from the recursive pattern matching scheme to a +# slower, iterative scheme in the event of a RecursionError. from ctypes import * STRING = c_char_p diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py index c0507cf3fbcbf..a265941490d5d 100644 --- a/Lib/lib2to3/tests/test_all_fixers.py +++ b/Lib/lib2to3/tests/test_all_fixers.py @@ -6,8 +6,10 @@ # Author: Collin Winter # Python imports -import unittest +import os.path +import sys import test.support +import unittest # Local imports from . import support @@ -19,9 +21,22 @@ class Test_all(support.TestCase): def setUp(self): self.refactor = support.get_refactorer() + def refactor_file(self, filepath): + if test.support.verbose: + print(f"Refactor file: {filepath}") + if os.path.basename(filepath) == 'infinite_recursion.py': + # bpo-46542: Processing infinite_recursion.py can crash Python + # if Python is built in debug mode: lower the recursion limit + # to prevent a crash. + with test.support.infinite_recursion(150): + self.refactor.refactor_file(filepath) + else: + self.refactor.refactor_file(filepath) + def test_all_project_files(self): for filepath in support.all_project_files(): - self.refactor.refactor_file(filepath) + with self.subTest(filepath=filepath): + self.refactor_file(filepath) if __name__ == '__main__': unittest.main() diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 034b5039cf00c..3884872437c47 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -20,6 +20,7 @@ import subprocess import sys import tempfile +import test.support import unittest # Local imports @@ -582,25 +583,31 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" + def parse_file(self, filepath): + if test.support.verbose: + print(f"Parse file: {filepath}") + with open(filepath, "rb") as fp: + encoding = tokenize.detect_encoding(fp.readline)[0] + self.assertIsNotNone(encoding, + "can't detect encoding for %s" % filepath) + with open(filepath, "r", encoding=encoding) as fp: + source = fp.read() + try: + tree = driver.parse_string(source) + except ParseError: + try: + tree = driver_no_print_statement.parse_string(source) + except ParseError as err: + self.fail('ParseError on file %s (%s)' % (filepath, err)) + new = str(tree) + if new != source: + print(diff_texts(source, new, filepath)) + self.fail("Idempotency failed: %s" % filepath) + def test_all_project_files(self): for filepath in support.all_project_files(): - with open(filepath, "rb") as fp: - encoding = tokenize.detect_encoding(fp.readline)[0] - self.assertIsNotNone(encoding, - "can't detect encoding for %s" % filepath) - with open(filepath, "r", encoding=encoding) as fp: - source = fp.read() - try: - tree = driver.parse_string(source) - except ParseError: - try: - tree = driver_no_print_statement.parse_string(source) - except ParseError as err: - self.fail('ParseError on file %s (%s)' % (filepath, err)) - new = str(tree) - if new != source: - print(diff_texts(source, new, filepath)) - self.fail("Idempotency failed: %s" % filepath) + with self.subTest(filepath=filepath): + self.parse_file(filepath) def test_extended_unpacking(self): driver.parse_string("a, *b, c = x\n") diff --git a/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst new file mode 100644 index 0000000000000..5596498724930 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst @@ -0,0 +1,2 @@ +Fix a Python crash in test_lib2to3 when using Python built in debug mode: +limit the recursion limit. Patch by Victor Stinner.