[Python-checkins] [3.9] bpo-45806: Fix recovery from stack overflow for 3.9. Again. (GH-29640)
ambv
webhook-mailer at python.org
Fri Nov 19 13:51:59 EST 2021
https://github.com/python/cpython/commit/4296396db017d782d3aa16100b366748c9ea4a04
commit: 4296396db017d782d3aa16100b366748c9ea4a04
branch: 3.9
author: Mark Shannon <mark at hotpy.org>
committer: ambv <lukasz at langa.pl>
date: 2021-11-19T19:51:50+01:00
summary:
[3.9] bpo-45806: Fix recovery from stack overflow for 3.9. Again. (GH-29640)
Co-authored-by: Łukasz Langa <lukasz at langa.pl>
files:
A Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst
M Include/internal/pycore_ceval.h
M Lib/test/test_exceptions.py
M Lib/test/test_sys.py
M Python/ceval.c
M Python/errors.c
M Python/sysmodule.c
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 18c8f027af16e..e7ace9bd01c2a 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -90,24 +90,8 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) {
#define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where)
-/* Compute the "lower-water mark" for a recursion limit. When
- * Py_LeaveRecursiveCall() is called with a recursion depth below this mark,
- * the overflowed flag is reset to 0. */
-static inline int _Py_RecursionLimitLowerWaterMark(int limit) {
- if (limit > 200) {
- return (limit - 50);
- }
- else {
- return (3 * (limit >> 2));
- }
-}
-
static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) {
tstate->recursion_depth--;
- int limit = tstate->interp->ceval.recursion_limit;
- if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) {
- tstate->overflowed = 0;
- }
}
static inline void _Py_LeaveRecursiveCall_inline(void) {
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 4a7ca60b39bc5..5168b0b8a0831 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -1213,7 +1213,7 @@ def gen():
# tstate->recursion_depth is equal to (recursion_limit - 1)
# and is equal to recursion_limit when _gen_throw() calls
# PyErr_NormalizeException().
- recurse(setrecursionlimit(depth + 2) - depth - 1)
+ recurse(setrecursionlimit(depth + 2) - depth)
finally:
sys.setrecursionlimit(recursionlimit)
print('Done.')
@@ -1243,6 +1243,52 @@ def test_recursion_normalizing_infinite_exception(self):
b'while normalizing an exception', err)
self.assertIn(b'Done.', out)
+ def test_recursion_in_except_handler(self):
+
+ def set_relative_recursion_limit(n):
+ depth = 1
+ while True:
+ try:
+ sys.setrecursionlimit(depth)
+ except RecursionError:
+ depth += 1
+ else:
+ break
+ sys.setrecursionlimit(depth+n)
+
+ def recurse_in_except():
+ try:
+ 1/0
+ except:
+ recurse_in_except()
+
+ def recurse_after_except():
+ try:
+ 1/0
+ except:
+ pass
+ recurse_after_except()
+
+ def recurse_in_body_and_except():
+ try:
+ recurse_in_body_and_except()
+ except:
+ recurse_in_body_and_except()
+
+ recursionlimit = sys.getrecursionlimit()
+ try:
+ set_relative_recursion_limit(10)
+ for func in (recurse_in_except, recurse_after_except, recurse_in_body_and_except):
+ with self.subTest(func=func):
+ try:
+ func()
+ except RecursionError:
+ pass
+ else:
+ self.fail("Should have raised a RecursionError")
+ finally:
+ sys.setrecursionlimit(recursionlimit)
+
@cpython_only
def test_recursion_normalizing_with_no_memory(self):
# Issue #30697. Test that in the abort that occurs when there is no
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 450e3920efa7c..2f1e5e971e8e9 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -260,42 +260,10 @@ def set_recursion_limit_at_depth(depth, limit):
sys.setrecursionlimit(1000)
for limit in (10, 25, 50, 75, 100, 150, 200):
- # formula extracted from _Py_RecursionLimitLowerWaterMark()
- if limit > 200:
- depth = limit - 50
- else:
- depth = limit * 3 // 4
- set_recursion_limit_at_depth(depth, limit)
+ set_recursion_limit_at_depth(limit, limit)
finally:
sys.setrecursionlimit(oldlimit)
- # The error message is specific to CPython
- @test.support.cpython_only
- def test_recursionlimit_fatalerror(self):
- # A fatal error occurs if a second recursion limit is hit when recovering
- # from a first one.
- code = textwrap.dedent("""
- import sys
-
- def f():
- try:
- f()
- except RecursionError:
- f()
-
- sys.setrecursionlimit(%d)
- f()""")
- with test.support.SuppressCrashReport():
- for i in (50, 1000):
- sub = subprocess.Popen([sys.executable, '-c', code % i],
- stderr=subprocess.PIPE)
- err = sub.communicate()[1]
- self.assertTrue(sub.returncode, sub.returncode)
- self.assertIn(
- b"Fatal Python error: _Py_CheckRecursiveCall: "
- b"Cannot recover from stack overflow",
- err)
-
def test_getwindowsversion(self):
# Raise SkipTest if sys doesn't have getwindowsversion attribute
test.support.get_attribute(sys, "getwindowsversion")
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst
new file mode 100644
index 0000000000000..f8c47cae4b0b9
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst
@@ -0,0 +1,5 @@
+Re-introduced fix that allows recovery from stack overflow without crashing
+the interpreter. The original fix as part of :issue:`42500` was reverted
+(see release notes for Python 3.9.4) since it introduced an ABI change in a
+bugfix release which is not allowed. The new fix doesn't introduce any ABI
+changes. Patch by Mark Shannon.
diff --git a/Python/ceval.c b/Python/ceval.c
index 86a5d81b7669a..9a61f8a3c3cf5 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -797,19 +797,21 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
/* Somebody asked that we don't check for recursion. */
return 0;
if (tstate->overflowed) {
- if (tstate->recursion_depth > recursion_limit + 50) {
+ if (tstate->recursion_depth > recursion_limit + 50 || tstate->overflowed > 50) {
/* Overflowing while handling an overflow. Give up. */
Py_FatalError("Cannot recover from stack overflow.");
}
- return 0;
}
- if (tstate->recursion_depth > recursion_limit) {
- --tstate->recursion_depth;
- tstate->overflowed = 1;
- _PyErr_Format(tstate, PyExc_RecursionError,
- "maximum recursion depth exceeded%s",
- where);
- return -1;
+ else {
+ if (tstate->recursion_depth > recursion_limit) {
+ tstate->overflowed++;
+ _PyErr_Format(tstate, PyExc_RecursionError,
+ "maximum recursion depth exceeded%s",
+ where);
+ tstate->overflowed--;
+ --tstate->recursion_depth;
+ return -1;
+ }
}
return 0;
}
diff --git a/Python/errors.c b/Python/errors.c
index 40d8e68bde783..79278764cc512 100644
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -317,12 +317,14 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
PyObject **val, PyObject **tb)
{
int recursion_depth = 0;
+ tstate->overflowed++;
PyObject *type, *value, *initial_tb;
restart:
type = *exc;
if (type == NULL) {
/* There was no exception, so nothing to do. */
+ tstate->overflowed--;
return;
}
@@ -374,6 +376,7 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
}
*exc = type;
*val = value;
+ tstate->overflowed--;
return;
error:
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 3e4115fe8e1f9..a52b299474c5d 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -17,7 +17,7 @@ Data members:
#include "Python.h"
#include "code.h"
#include "frameobject.h" // PyFrame_GetBack()
-#include "pycore_ceval.h" // _Py_RecursionLimitLowerWaterMark()
+#include "pycore_ceval.h"
#include "pycore_initconfig.h"
#include "pycore_object.h"
#include "pycore_pathconfig.h"
@@ -1160,7 +1160,6 @@ static PyObject *
sys_setrecursionlimit_impl(PyObject *module, int new_limit)
/*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/
{
- int mark;
PyThreadState *tstate = _PyThreadState_GET();
if (new_limit < 1) {
@@ -1178,8 +1177,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
Reject too low new limit if the current recursion depth is higher than
the new low-water mark. Otherwise it may not be possible anymore to
reset the overflowed flag to 0. */
- mark = _Py_RecursionLimitLowerWaterMark(new_limit);
- if (tstate->recursion_depth >= mark) {
+ if (tstate->recursion_depth >= new_limit) {
_PyErr_Format(tstate, PyExc_RecursionError,
"cannot set the recursion limit to %i at "
"the recursion depth %i: the limit is too low",
More information about the Python-checkins
mailing list