[Python-checkins] bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035)
ambv
webhook-mailer at python.org
Mon Jan 31 12:46:24 EST 2022
https://github.com/python/cpython/commit/ee0ac328d38a86f7907598c94cb88a97635b32f8
commit: ee0ac328d38a86f7907598c94cb88a97635b32f8
branch: main
author: Victor Stinner <vstinner at python.org>
committer: ambv <lukasz at langa.pl>
date: 2022-01-31T18:46:09+01:00
summary:
bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035)
* bpo-46542: test_lib2to3 uses support.infinite_recursion()
Fix a Python crash in test_lib2to3 when using Python built in debug
mode: limit the recursion limit.
The test_all_project_files() test of test_lib2to3 now uses the
test.support.infinite_recursion() context manager when processing the
infinite_recursion.py file to prevent a crash when Python is built in
debug mode.
The two test_all_project_files() tests now use subTest() and log the
refactored/parsed filename (if test_lib2to3 is run in verbose mode).
* Update Lib/lib2to3/tests/data/infinite_recursion.py
Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com>
Co-authored-by: Łukasz Langa <lukasz at langa.pl>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com>
files:
A Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst
M Lib/lib2to3/pytree.py
M Lib/lib2to3/tests/data/infinite_recursion.py
M Lib/lib2to3/tests/test_all_fixers.py
M Lib/lib2to3/tests/test_parser.py
diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py
index 2a6ef2ef5240a..729023df0284e 100644
--- a/Lib/lib2to3/pytree.py
+++ b/Lib/lib2to3/pytree.py
@@ -720,8 +720,8 @@ def generate_matches(self, nodes):
r[self.name] = nodes[:count]
yield count, r
except RuntimeError:
- # We fall back to the iterative pattern matching scheme if the recursive
- # scheme hits the recursion limit.
+ # Fall back to the iterative pattern matching scheme if the
+ # recursive scheme hits the recursion limit (RecursionError).
for count, r in self._iterative_matches(nodes):
if self.name:
r[self.name] = nodes[:count]
diff --git a/Lib/lib2to3/tests/data/infinite_recursion.py b/Lib/lib2to3/tests/data/infinite_recursion.py
index 71715ef29dc48..acc62ed4f1a44 100644
--- a/Lib/lib2to3/tests/data/infinite_recursion.py
+++ b/Lib/lib2to3/tests/data/infinite_recursion.py
@@ -1,5 +1,5 @@
-# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching
-# scheme in the event that the faster recursive system fails due to infinite recursion.
+# Verify that 2to3 falls back from the recursive pattern matching scheme to a
+# slower, iterative scheme in the event of a RecursionError.
from ctypes import *
STRING = c_char_p
diff --git a/Lib/lib2to3/tests/test_all_fixers.py b/Lib/lib2to3/tests/test_all_fixers.py
index c0507cf3fbcbf..a265941490d5d 100644
--- a/Lib/lib2to3/tests/test_all_fixers.py
+++ b/Lib/lib2to3/tests/test_all_fixers.py
@@ -6,8 +6,10 @@
# Author: Collin Winter
# Python imports
-import unittest
+import os.path
+import sys
import test.support
+import unittest
# Local imports
from . import support
@@ -19,9 +21,22 @@ class Test_all(support.TestCase):
def setUp(self):
self.refactor = support.get_refactorer()
+ def refactor_file(self, filepath):
+ if test.support.verbose:
+ print(f"Refactor file: {filepath}")
+ if os.path.basename(filepath) == 'infinite_recursion.py':
+ # bpo-46542: Processing infinite_recursion.py can crash Python
+ # if Python is built in debug mode: lower the recursion limit
+ # to prevent a crash.
+ with test.support.infinite_recursion(150):
+ self.refactor.refactor_file(filepath)
+ else:
+ self.refactor.refactor_file(filepath)
+
def test_all_project_files(self):
for filepath in support.all_project_files():
- self.refactor.refactor_file(filepath)
+ with self.subTest(filepath=filepath):
+ self.refactor_file(filepath)
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index ff4f8078878d8..74a5787574209 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -20,6 +20,7 @@
import subprocess
import sys
import tempfile
+import test.support
import unittest
# Local imports
@@ -589,25 +590,31 @@ class TestParserIdempotency(support.TestCase):
"""A cut-down version of pytree_idempotency.py."""
+ def parse_file(self, filepath):
+ if test.support.verbose:
+ print(f"Parse file: {filepath}")
+ with open(filepath, "rb") as fp:
+ encoding = tokenize.detect_encoding(fp.readline)[0]
+ self.assertIsNotNone(encoding,
+ "can't detect encoding for %s" % filepath)
+ with open(filepath, "r", encoding=encoding) as fp:
+ source = fp.read()
+ try:
+ tree = driver.parse_string(source)
+ except ParseError:
+ try:
+ tree = driver_no_print_statement.parse_string(source)
+ except ParseError as err:
+ self.fail('ParseError on file %s (%s)' % (filepath, err))
+ new = str(tree)
+ if new != source:
+ print(diff_texts(source, new, filepath))
+ self.fail("Idempotency failed: %s" % filepath)
+
def test_all_project_files(self):
for filepath in support.all_project_files():
- with open(filepath, "rb") as fp:
- encoding = tokenize.detect_encoding(fp.readline)[0]
- self.assertIsNotNone(encoding,
- "can't detect encoding for %s" % filepath)
- with open(filepath, "r", encoding=encoding) as fp:
- source = fp.read()
- try:
- tree = driver.parse_string(source)
- except ParseError:
- try:
- tree = driver_no_print_statement.parse_string(source)
- except ParseError as err:
- self.fail('ParseError on file %s (%s)' % (filepath, err))
- new = str(tree)
- if new != source:
- print(diff_texts(source, new, filepath))
- self.fail("Idempotency failed: %s" % filepath)
+ with self.subTest(filepath=filepath):
+ self.parse_file(filepath)
def test_extended_unpacking(self):
driver.parse_string("a, *b, c = x\n")
diff --git a/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst
new file mode 100644
index 0000000000000..5596498724930
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst
@@ -0,0 +1,2 @@
+Fix a Python crash in test_lib2to3 when using Python built in debug mode:
+limit the recursion limit. Patch by Victor Stinner.
More information about the Python-checkins
mailing list