Python-checkins
Threads by month
- ----- 2024 -----
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2010 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2009 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2008 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2007 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2006 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2005 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2004 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2003 -----
- December
- November
- October
- September
- August
December 2021
- 1 participants
- 354 discussions
01 Dec '21
https://github.com/python/cpython/commit/f27bef30438d2f07f19de91e021f34b77c…
commit: f27bef30438d2f07f19de91e021f34b77ccc4b20
branch: main
author: Rob <r4226(a)pm.me>
committer: asvetlov <andrew.svetlov(a)gmail.com>
date: 2021-12-01T14:24:46+02:00
summary:
bpo-45896: Fix docs default asyncio event loop on Windows (GH-29857)
files:
M Doc/library/asyncio-eventloop.rst
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 5a3369415a745..497128ee17f37 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1243,9 +1243,10 @@ async/await code consider using the high-level
.. note::
- The default asyncio event loop on **Windows** does not support
- subprocesses. See :ref:`Subprocess Support on Windows
- <asyncio-windows-subprocess>` for details.
+ On Windows, the default event loop :class:`ProactorEventLoop` supports
+ subprocesses, whereas :class:`SelectorEventLoop` does not. See
+ :ref:`Subprocess Support on Windows <asyncio-windows-subprocess>` for
+ details.
.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \
stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
1
0
https://github.com/python/cpython/commit/49444fb807ecb396462c8e5f547eeb5c6b…
commit: 49444fb807ecb396462c8e5f547eeb5c6bc5d4de
branch: main
author: Mark Shannon <mark(a)hotpy.org>
committer: markshannon <mark(a)hotpy.org>
date: 2021-12-01T12:09:36Z
summary:
bpo-45753: Interpreter internal tweaks (GH-29575)
* Split exit paths into exceptional and non-exceptional.
* Move exit tracing code to individual bytecodes.
* Wrap all trace entry and exit events in macros to make them clearer and easier to enhance.
* Move return sequence into RETURN_VALUE, YIELD_VALUE and YIELD_FROM. Distinguish between normal trace events and dtrace events.
files:
M Include/internal/pycore_code.h
M Include/internal/pycore_frame.h
M Python/ceval.c
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 194af46a3a274..d4d1392d05bde 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -137,24 +137,25 @@ _GetSpecializedCacheEntryForInstruction(const _Py_CODEUNIT *first_instr, int nex
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
#define QUICKENING_WARMUP_COLDEST 1
-static inline void
-PyCodeObject_IncrementWarmup(PyCodeObject * co)
-{
- co->co_warmup++;
-}
+int _Py_Quicken(PyCodeObject *code);
-/* Used by the interpreter to determine when a code object should be quickened */
+/* Returns 1 if quickening occurs.
+ * -1 if an error occurs
+ * 0 otherwise */
static inline int
-PyCodeObject_IsWarmedUp(PyCodeObject * co)
+_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
{
- return (co->co_warmup == 0);
+ if (code->co_warmup != 0) {
+ code->co_warmup++;
+ if (code->co_warmup == 0) {
+ return _Py_Quicken(code) ? -1 : 1;
+ }
+ }
+ return 0;
}
-int _Py_Quicken(PyCodeObject *code);
-
extern Py_ssize_t _Py_QuickenedCount;
-
/* "Locals plus" for a code object is the set of locals + cell vars +
* free vars. This relates to variable names as well as offsets into
* the "fast locals" storage array of execution frames. The compiler
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index 0015de8f8fcb4..b0e51a6ddc461 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -19,6 +19,11 @@ enum _framestate {
typedef signed char PyFrameState;
+/*
+ frame->f_lasti refers to the index of the last instruction,
+ unless it's -1 in which case next_instr should be first_instr.
+*/
+
typedef struct _interpreter_frame {
PyFunctionObject *f_func; /* Strong reference */
PyObject *f_globals; /* Borrowed reference */
diff --git a/Python/ceval.c b/Python/ceval.c
index c5477b30f7e56..97c684479abdc 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -102,8 +102,8 @@ static InterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, PyObject* const* args,
size_t argcount, PyObject *kwnames);
-static int
-_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame);
+static void
+_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame *frame);
#define NAME_ERROR_MSG \
"name '%.200s' is not defined"
@@ -1390,7 +1390,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
/* Get opcode and oparg from original instructions, not quickened form. */
#define TRACING_NEXTOPARG() do { \
- _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \
+ _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
opcode = _Py_OPCODE(word); \
oparg = _Py_OPARG(word); \
} while (0)
@@ -1462,20 +1462,20 @@ eval_frame_handle_pending(PyThreadState *tstate)
#ifdef LLTRACE
#define PUSH(v) { (void)(BASIC_PUSH(v), \
lltrace && prtrace(tstate, TOP(), "push")); \
- assert(STACK_LEVEL() <= co->co_stacksize); }
+ assert(STACK_LEVEL() <= frame->f_code->co_stacksize); }
#define POP() ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \
BASIC_POP())
#define STACK_GROW(n) do { \
assert(n >= 0); \
(void)(BASIC_STACKADJ(n), \
lltrace && prtrace(tstate, TOP(), "stackadj")); \
- assert(STACK_LEVEL() <= co->co_stacksize); \
+ assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
} while (0)
#define STACK_SHRINK(n) do { \
assert(n >= 0); \
(void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \
(void)(BASIC_STACKADJ(-(n))); \
- assert(STACK_LEVEL() <= co->co_stacksize); \
+ assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
} while (0)
#define EXT_POP(STACK_POINTER) ((void)(lltrace && \
prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \
@@ -1537,6 +1537,40 @@ eval_frame_handle_pending(PyThreadState *tstate)
STAT_INC(LOAD_##attr_or_method, hit); \
Py_INCREF(res);
+#define TRACE_FUNCTION_EXIT() \
+ if (cframe.use_tracing) { \
+ if (trace_function_exit(tstate, frame, retval)) { \
+ Py_DECREF(retval); \
+ goto exit_unwind; \
+ } \
+ }
+
+#define DTRACE_FUNCTION_EXIT() \
+ if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \
+ dtrace_function_return(frame); \
+ }
+
+#define TRACE_FUNCTION_UNWIND() \
+ if (cframe.use_tracing) { \
+ /* Since we are already unwinding, \
+ * we dont't care if this raises */ \
+ trace_function_exit(tstate, frame, NULL); \
+ }
+
+#define TRACE_FUNCTION_ENTRY() \
+ if (cframe.use_tracing) { \
+ if (trace_function_entry(tstate, frame)) { \
+ goto exit_unwind; \
+ } \
+ }
+
+#define DTRACE_FUNCTION_ENTRY() \
+ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \
+ dtrace_function_entry(frame); \
+ }
+
+
+
static int
trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
{
@@ -1576,6 +1610,24 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
return 0;
}
+static int
+trace_function_exit(PyThreadState *tstate, InterpreterFrame *frame, PyObject *retval)
+{
+ if (tstate->c_tracefunc) {
+ if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
+ tstate, frame, PyTrace_RETURN, retval)) {
+ return -1;
+ }
+ }
+ if (tstate->c_profilefunc) {
+ if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
+ tstate, frame, PyTrace_RETURN, retval)) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
static PyObject *
make_coro(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals,
@@ -1583,7 +1635,8 @@ make_coro(PyThreadState *tstate, PyFunctionObject *func,
PyObject *kwnames);
static int
-skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
+skip_backwards_over_extended_args(PyCodeObject *code, int offset)
+{
_Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) {
offset--;
@@ -1591,6 +1644,14 @@ skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
return offset;
}
+static InterpreterFrame *
+pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
+{
+ InterpreterFrame *prev_frame = frame->previous;
+ _PyEvalFrameClearAndPop(tstate, frame);
+ return prev_frame;
+}
+
PyObject* _Py_HOT_FUNCTION
_PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
{
@@ -1606,7 +1667,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
#endif
int opcode; /* Current opcode */
int oparg; /* Current opcode argument, if any */
- PyObject *retval = NULL; /* Return value */
_Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
CFrame cframe;
@@ -1625,82 +1685,77 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
frame->previous = prev_cframe->current_frame;
cframe.current_frame = frame;
+ /* support for generator.throw() */
+ if (throwflag) {
+ if (_Py_EnterRecursiveCall(tstate, "")) {
+ tstate->recursion_remaining--;
+ goto exit_unwind;
+ }
+ TRACE_FUNCTION_ENTRY();
+ DTRACE_FUNCTION_ENTRY();
+ goto resume_with_error;
+ }
+
+ /* Local "register" variables.
+ * These are cached values from the frame and code object. */
+
+ PyObject *names;
+ PyObject *consts;
+ _Py_CODEUNIT *first_instr;
+ _Py_CODEUNIT *next_instr;
+ PyObject **stack_pointer;
+
+/* Sets the above local variables from the frame */
+#define SET_LOCALS_FROM_FRAME() \
+ { \
+ PyCodeObject *co = frame->f_code; \
+ names = co->co_names; \
+ consts = co->co_consts; \
+ first_instr = co->co_firstinstr; \
+ } \
+ assert(frame->f_lasti >= -1); \
+ next_instr = first_instr + frame->f_lasti + 1; \
+ stack_pointer = _PyFrame_GetStackPointer(frame); \
+ /* Set stackdepth to -1. \
+ Update when returning or calling trace function. \
+ Having stackdepth <= 0 ensures that invalid \
+ values are not visible to the cycle GC. \
+ We choose -1 rather than 0 to assist debugging. \
+ */ \
+ frame->stacktop = -1;
+
+
start_frame:
if (_Py_EnterRecursiveCall(tstate, "")) {
tstate->recursion_remaining--;
- goto exit_eval_frame;
+ goto exit_unwind;
}
assert(tstate->cframe == &cframe);
assert(frame == cframe.current_frame);
- if (cframe.use_tracing) {
- if (trace_function_entry(tstate, frame)) {
- goto exit_eval_frame;
- }
- }
-
- if (PyDTrace_FUNCTION_ENTRY_ENABLED())
- dtrace_function_entry(frame);
+ TRACE_FUNCTION_ENTRY();
+ DTRACE_FUNCTION_ENTRY();
- PyCodeObject *co = frame->f_code;
- /* Increment the warmup counter and quicken if warm enough
- * _Py_Quicken is idempotent so we don't worry about overflow */
- if (!PyCodeObject_IsWarmedUp(co)) {
- PyCodeObject_IncrementWarmup(co);
- if (PyCodeObject_IsWarmedUp(co)) {
- if (_Py_Quicken(co)) {
- goto exit_eval_frame;
- }
- }
+ if (_Py_IncrementCountAndMaybeQuicken(frame->f_code) < 0) {
+ goto exit_unwind;
}
+ frame->f_state = FRAME_EXECUTING;
resume_frame:
- co = frame->f_code;
- PyObject *names = co->co_names;
- PyObject *consts = co->co_consts;
- _Py_CODEUNIT *first_instr = co->co_firstinstr;
- /*
- frame->f_lasti refers to the index of the last instruction,
- unless it's -1 in which case next_instr should be first_instr.
-
- YIELD_FROM sets frame->f_lasti to itself, in order to repeatedly yield
- multiple values.
-
- When the PREDICT() macros are enabled, some opcode pairs follow in
- direct succession. A successful prediction effectively links the two
- codes together as if they were a single new opcode, but the value
- of frame->f_lasti is correctly updated so potential inlined calls
- or lookups of frame->f_lasti are aways correct when the macros are used.
- */
- assert(frame->f_lasti >= -1);
- _Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1;
- PyObject **stack_pointer = _PyFrame_GetStackPointer(frame);
- /* Set stackdepth to -1.
- * Update when returning or calling trace function.
- Having stackdepth <= 0 ensures that invalid
- values are not visible to the cycle GC.
- We choose -1 rather than 0 to assist debugging.
- */
- frame->stacktop = -1;
- frame->f_state = FRAME_EXECUTING;
+ SET_LOCALS_FROM_FRAME();
#ifdef LLTRACE
_Py_IDENTIFIER(__ltrace__);
{
int r = _PyDict_ContainsId(GLOBALS(), &PyId___ltrace__);
if (r < 0) {
- goto exit_eval_frame;
+ goto exit_unwind;
}
lltrace = r;
}
#endif
- if (throwflag) { /* support for generator.throw() */
- throwflag = 0;
- goto error;
- }
-
#ifdef Py_DEBUG
/* _PyEval_EvalFrameDefault() must not be called with an exception set,
because it can clear it (directly or indirectly) and so the
@@ -1711,7 +1766,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
check_eval_breaker:
{
assert(STACK_LEVEL() >= 0); /* else underflow */
- assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */
+ assert(STACK_LEVEL() <= frame->f_code->co_stacksize); /* else overflow */
assert(!_PyErr_Occurred(tstate));
/* Do periodic things. Doing this every time through
@@ -2418,11 +2473,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
}
TARGET(RETURN_VALUE) {
- retval = POP();
+ PyObject *retval = POP();
assert(EMPTY());
frame->f_state = FRAME_RETURNED;
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exiting;
+ TRACE_FUNCTION_EXIT();
+ DTRACE_FUNCTION_EXIT();
+ _Py_LeaveRecursiveCall(tstate);
+ if (frame->depth) {
+ frame = cframe.current_frame = pop_frame(tstate, frame);
+ _PyFrame_StackPush(frame, retval);
+ goto resume_frame;
+ }
+ /* Restore previous cframe and return. */
+ tstate->cframe = cframe.previous;
+ tstate->cframe->use_tracing = cframe.use_tracing;
+ assert(tstate->cframe->current_frame == frame->previous);
+ assert(!_PyErr_Occurred(tstate));
+ return retval;
}
TARGET(GET_AITER) {
@@ -2562,9 +2630,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
}
TARGET(YIELD_FROM) {
+ assert(frame->depth == 0);
PyObject *v = POP();
PyObject *receiver = TOP();
PySendResult gen_status;
+ PyObject *retval;
if (tstate->c_tracefunc == NULL) {
gen_status = PyIter_Send(receiver, v, &retval);
} else {
@@ -2610,13 +2680,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
frame->f_lasti -= 1;
frame->f_state = FRAME_SUSPENDED;
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exiting;
+ TRACE_FUNCTION_EXIT();
+ DTRACE_FUNCTION_EXIT();
+ _Py_LeaveRecursiveCall(tstate);
+ /* Restore previous cframe and return. */
+ tstate->cframe = cframe.previous;
+ tstate->cframe->use_tracing = cframe.use_tracing;
+ assert(tstate->cframe->current_frame == frame->previous);
+ assert(!_PyErr_Occurred(tstate));
+ return retval;
}
TARGET(YIELD_VALUE) {
- retval = POP();
+ assert(frame->depth == 0);
+ PyObject *retval = POP();
- if (co->co_flags & CO_ASYNC_GENERATOR) {
+ if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) {
PyObject *w = _PyAsyncGenValueWrapperNew(retval);
Py_DECREF(retval);
if (w == NULL) {
@@ -2627,7 +2706,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
}
frame->f_state = FRAME_SUSPENDED;
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exiting;
+ TRACE_FUNCTION_EXIT();
+ DTRACE_FUNCTION_EXIT();
+ _Py_LeaveRecursiveCall(tstate);
+ /* Restore previous cframe and return. */
+ tstate->cframe = cframe.previous;
+ tstate->cframe->use_tracing = cframe.use_tracing;
+ assert(tstate->cframe->current_frame == frame->previous);
+ assert(!_PyErr_Occurred(tstate));
+ return retval;
}
TARGET(GEN_START) {
@@ -3100,7 +3187,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
format_exc_check_arg(
tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(co->co_localsplusnames, oparg)
+ PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
);
goto error;
}
@@ -3125,15 +3212,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
Py_DECREF(oldobj);
DISPATCH();
}
- format_exc_unbound(tstate, co, oparg);
+ format_exc_unbound(tstate, frame->f_code, oparg);
goto error;
}
TARGET(LOAD_CLASSDEREF) {
PyObject *name, *value, *locals = LOCALS();
assert(locals);
- assert(oparg >= 0 && oparg < co->co_nlocalsplus);
- name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
+ assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
+ name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
if (PyDict_CheckExact(locals)) {
value = PyDict_GetItemWithError(locals, name);
if (value != NULL) {
@@ -3156,7 +3243,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, co, oparg);
+ format_exc_unbound(tstate, frame->f_code, oparg);
goto error;
}
Py_INCREF(value);
@@ -3169,7 +3256,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
PyObject *cell = GETLOCAL(oparg);
PyObject *value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, co, oparg);
+ format_exc_unbound(tstate, frame->f_code, oparg);
goto error;
}
Py_INCREF(value);
@@ -3918,18 +4005,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
TARGET(JUMP_ABSOLUTE) {
PREDICTED(JUMP_ABSOLUTE);
assert(oparg < INSTR_OFFSET());
- /* Increment the warmup counter and quicken if warm enough
- * _Py_Quicken is idempotent so we don't worry about overflow */
- if (!PyCodeObject_IsWarmedUp(co)) {
- PyCodeObject_IncrementWarmup(co);
- if (PyCodeObject_IsWarmedUp(co)) {
- if (_Py_Quicken(co)) {
- goto error;
- }
- int nexti = INSTR_OFFSET();
- first_instr = co->co_firstinstr;
- next_instr = first_instr + nexti;
+ int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
+ if (err) {
+ if (err < 0) {
+ goto error;
}
+ /* Update first_instr and next_instr to point to newly quickened code */
+ int nexti = INSTR_OFFSET();
+ first_instr = frame->f_code->co_firstinstr;
+ next_instr = first_instr + nexti;
}
JUMPTO(oparg);
CHECK_EVAL_BREAKER();
@@ -4036,7 +4120,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
PyObject *iter;
if (PyCoro_CheckExact(iterable)) {
/* `iterable` is a coroutine */
- if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
+ if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
regular generator. */
Py_DECREF(iterable);
@@ -4906,7 +4990,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
#else
case DO_TRACING: {
#endif
- int instr_prev = skip_backwards_over_extended_args(co, frame->f_lasti);
+ int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti);
frame->f_lasti = INSTR_OFFSET();
TRACING_NEXTOPARG();
if (PyDTrace_LINE_ENABLED()) {
@@ -5016,7 +5100,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
{
format_exc_check_arg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(co->co_localsplusnames, oparg)
+ PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
);
goto error;
}
@@ -5039,8 +5123,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
}
if (tstate->c_tracefunc != NULL) {
- /* Make sure state is set to FRAME_EXECUTING for tracing */
- assert(frame->f_state == FRAME_EXECUTING);
+ /* Make sure state is set to FRAME_UNWINDING for tracing */
frame->f_state = FRAME_UNWINDING;
call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
tstate, frame);
@@ -5051,9 +5134,8 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
/* We can't use frame->f_lasti here, as RERAISE may have set it */
int offset = INSTR_OFFSET()-1;
int level, handler, lasti;
- if (get_exception_handler(co, offset, &level, &handler, &lasti) == 0) {
+ if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
// No handlers, so exit.
- assert(retval == NULL);
assert(_PyErr_Occurred(tstate));
/* Pop remaining stack entries. */
@@ -5065,7 +5147,9 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
assert(STACK_LEVEL() == 0);
_PyFrame_SetStackPointer(frame, stack_pointer);
frame->f_state = FRAME_RAISED;
- goto exiting;
+ TRACE_FUNCTION_UNWIND();
+ DTRACE_FUNCTION_EXIT();
+ goto exit_unwind;
}
assert(STACK_LEVEL() >= level);
@@ -5106,48 +5190,22 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
DISPATCH();
}
-exiting:
- if (cframe.use_tracing) {
- if (tstate->c_tracefunc) {
- if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
- tstate, frame, PyTrace_RETURN, retval)) {
- Py_CLEAR(retval);
- }
- }
- if (tstate->c_profilefunc) {
- if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
- tstate, frame, PyTrace_RETURN, retval)) {
- Py_CLEAR(retval);
- }
- }
- }
-
- /* pop frame */
-exit_eval_frame:
- if (PyDTrace_FUNCTION_RETURN_ENABLED())
- dtrace_function_return(frame);
+exit_unwind:
+ assert(_PyErr_Occurred(tstate));
_Py_LeaveRecursiveCall(tstate);
-
- if (frame->depth) {
- cframe.current_frame = frame->previous;
- _PyFrame_StackPush(cframe.current_frame, retval);
- if (_PyEvalFrameClearAndPop(tstate, frame)) {
- retval = NULL;
- }
- frame = cframe.current_frame;
- if (retval == NULL) {
- assert(_PyErr_Occurred(tstate));
- throwflag = 1;
- }
- retval = NULL;
- goto resume_frame;
+ if (frame->depth == 0) {
+ /* Restore previous cframe and exit */
+ tstate->cframe = cframe.previous;
+ tstate->cframe->use_tracing = cframe.use_tracing;
+ assert(tstate->cframe->current_frame == frame->previous);
+ return NULL;
}
+ frame = cframe.current_frame = pop_frame(tstate, frame);
+
+resume_with_error:
+ SET_LOCALS_FROM_FRAME();
+ goto error;
- /* Restore previous cframe. */
- tstate->cframe = cframe.previous;
- tstate->cframe->use_tracing = cframe.use_tracing;
- assert(tstate->cframe->current_frame == frame->previous);
- return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
}
static void
@@ -5770,15 +5828,14 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
return NULL;
}
-static int
+static void
_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame)
{
- --tstate->recursion_remaining;
+ tstate->recursion_remaining--;
assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0);
_PyFrame_Clear(frame);
- ++tstate->recursion_remaining;
+ tstate->recursion_remaining++;
_PyThreadState_PopFrame(tstate, frame);
- return 0;
}
PyObject *
@@ -5812,9 +5869,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
}
PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame));
- if (_PyEvalFrameClearAndPop(tstate, frame)) {
- retval = NULL;
- }
+ _PyEvalFrameClearAndPop(tstate, frame);
return retval;
}
1
0
30 Nov '21
https://github.com/python/cpython/commit/0aa0bd056349f73de9577ccc38560c1d01…
commit: 0aa0bd056349f73de9577ccc38560c1d01864d51
branch: main
author: Raymond Hettinger <rhettinger(a)users.noreply.github.com>
committer: rhettinger <rhettinger(a)users.noreply.github.com>
date: 2021-11-30T19:25:57-06:00
summary:
bpo-45876: Have stdev() also use decimal specific square root. (GH-29869)
files:
M Lib/statistics.py
diff --git a/Lib/statistics.py b/Lib/statistics.py
index 9f1efa21b15e3..ff19ce9672d33 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -920,9 +920,8 @@ def stdev(data, xbar=None):
raise StatisticsError('stdev requires at least two data points')
T, ss = _ss(data, xbar)
mss = ss / (n - 1)
- if hasattr(T, 'sqrt'):
- var = _convert(mss, T)
- return var.sqrt()
+ if issubclass(T, Decimal):
+ return _decimal_sqrt_of_frac(mss.numerator, mss.denominator)
return _float_sqrt_of_frac(mss.numerator, mss.denominator)
1
0
bpo-45876: Correctly rounded stdev() and pstdev() for the Decimal case (GH-29828)
by rhettinger 30 Nov '21
by rhettinger 30 Nov '21
30 Nov '21
https://github.com/python/cpython/commit/a39f46afdead515e7ac3722464b5ee8d7b…
commit: a39f46afdead515e7ac3722464b5ee8d7b0b2c9b
branch: main
author: Raymond Hettinger <rhettinger(a)users.noreply.github.com>
committer: rhettinger <rhettinger(a)users.noreply.github.com>
date: 2021-11-30T18:20:08-06:00
summary:
bpo-45876: Correctly rounded stdev() and pstdev() for the Decimal case (GH-29828)
files:
M Lib/statistics.py
M Lib/test/test_statistics.py
diff --git a/Lib/statistics.py b/Lib/statistics.py
index cf8eaa0a61e62..9f1efa21b15e3 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -137,7 +137,7 @@
from itertools import groupby, repeat
from bisect import bisect_left, bisect_right
from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum
-from operator import itemgetter, mul
+from operator import mul
from collections import Counter, namedtuple
_SQRT2 = sqrt(2.0)
@@ -248,6 +248,28 @@ def _exact_ratio(x):
x is expected to be an int, Fraction, Decimal or float.
"""
+
+ # XXX We should revisit whether using fractions to accumulate exact
+ # ratios is the right way to go.
+
+ # The integer ratios for binary floats can have numerators or
+ # denominators with over 300 decimal digits. The problem is more
+ # acute with decimal floats where the the default decimal context
+ # supports a huge range of exponents from Emin=-999999 to
+ # Emax=999999. When expanded with as_integer_ratio(), numbers like
+ # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large
+ # numerators or denominators that will slow computation.
+
+ # When the integer ratios are accumulated as fractions, the size
+ # grows to cover the full range from the smallest magnitude to the
+ # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300),
+ # has a 616 digit numerator. Likewise,
+ # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000'))
+ # has 10,003 digit numerator.
+
+ # This doesn't seem to have been problem in practice, but it is a
+ # potential pitfall.
+
try:
return x.as_integer_ratio()
except AttributeError:
@@ -305,28 +327,60 @@ def _fail_neg(values, errmsg='negative value'):
raise StatisticsError(errmsg)
yield x
-def _isqrt_frac_rto(n: int, m: int) -> float:
+
+def _integer_sqrt_of_frac_rto(n: int, m: int) -> int:
"""Square root of n/m, rounded to the nearest integer using round-to-odd."""
# Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf
a = math.isqrt(n // m)
return a | (a*a*m != n)
-# For 53 bit precision floats, the _sqrt_frac() shift is 109.
-_sqrt_shift: int = 2 * sys.float_info.mant_dig + 3
-def _sqrt_frac(n: int, m: int) -> float:
+# For 53 bit precision floats, the bit width used in
+# _float_sqrt_of_frac() is 109.
+_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3
+
+
+def _float_sqrt_of_frac(n: int, m: int) -> float:
"""Square root of n/m as a float, correctly rounded."""
# See principle and proof sketch at: https://bugs.python.org/msg407078
- q = (n.bit_length() - m.bit_length() - _sqrt_shift) // 2
+ q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2
if q >= 0:
- numerator = _isqrt_frac_rto(n, m << 2 * q) << q
+ numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q
denominator = 1
else:
- numerator = _isqrt_frac_rto(n << -2 * q, m)
+ numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m)
denominator = 1 << -q
return numerator / denominator # Convert to float
+def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal:
+ """Square root of n/m as a Decimal, correctly rounded."""
+ # Premise: For decimal, computing (n/m).sqrt() can be off
+ # by 1 ulp from the correctly rounded result.
+ # Method: Check the result, moving up or down a step if needed.
+ if n <= 0:
+ if not n:
+ return Decimal('0.0')
+ n, m = -n, -m
+
+ root = (Decimal(n) / Decimal(m)).sqrt()
+ nr, dr = root.as_integer_ratio()
+
+ plus = root.next_plus()
+ np, dp = plus.as_integer_ratio()
+ # test: n / m > ((root + plus) / 2) ** 2
+ if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2:
+ return plus
+
+ minus = root.next_minus()
+ nm, dm = minus.as_integer_ratio()
+ # test: n / m < ((root + minus) / 2) ** 2
+ if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2:
+ return minus
+
+ return root
+
+
# === Measures of central tendency (averages) ===
def mean(data):
@@ -869,7 +923,7 @@ def stdev(data, xbar=None):
if hasattr(T, 'sqrt'):
var = _convert(mss, T)
return var.sqrt()
- return _sqrt_frac(mss.numerator, mss.denominator)
+ return _float_sqrt_of_frac(mss.numerator, mss.denominator)
def pstdev(data, mu=None):
@@ -888,10 +942,9 @@ def pstdev(data, mu=None):
raise StatisticsError('pstdev requires at least one data point')
T, ss = _ss(data, mu)
mss = ss / n
- if hasattr(T, 'sqrt'):
- var = _convert(mss, T)
- return var.sqrt()
- return _sqrt_frac(mss.numerator, mss.denominator)
+ if issubclass(T, Decimal):
+ return _decimal_sqrt_of_frac(mss.numerator, mss.denominator)
+ return _float_sqrt_of_frac(mss.numerator, mss.denominator)
# === Statistics for relations between two inputs ===
diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py
index 771a03e707ee0..bacb76a9b036b 100644
--- a/Lib/test/test_statistics.py
+++ b/Lib/test/test_statistics.py
@@ -2164,9 +2164,9 @@ def test_center_not_at_mean(self):
class TestSqrtHelpers(unittest.TestCase):
- def test_isqrt_frac_rto(self):
+ def test_integer_sqrt_of_frac_rto(self):
for n, m in itertools.product(range(100), range(1, 1000)):
- r = statistics._isqrt_frac_rto(n, m)
+ r = statistics._integer_sqrt_of_frac_rto(n, m)
self.assertIsInstance(r, int)
if r*r*m == n:
# Root is exact
@@ -2177,7 +2177,7 @@ def test_isqrt_frac_rto(self):
self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2)
@requires_IEEE_754
- def test_sqrt_frac(self):
+ def test_float_sqrt_of_frac(self):
def is_root_correctly_rounded(x: Fraction, root: float) -> bool:
if not x:
@@ -2204,22 +2204,59 @@ def is_root_correctly_rounded(x: Fraction, root: float) -> bool:
denonimator: int = randrange(10 ** randrange(50)) + 1
with self.subTest(numerator=numerator, denonimator=denonimator):
x: Fraction = Fraction(numerator, denonimator)
- root: float = statistics._sqrt_frac(numerator, denonimator)
+ root: float = statistics._float_sqrt_of_frac(numerator, denonimator)
self.assertTrue(is_root_correctly_rounded(x, root))
# Verify that corner cases and error handling match math.sqrt()
- self.assertEqual(statistics._sqrt_frac(0, 1), 0.0)
+ self.assertEqual(statistics._float_sqrt_of_frac(0, 1), 0.0)
with self.assertRaises(ValueError):
- statistics._sqrt_frac(-1, 1)
+ statistics._float_sqrt_of_frac(-1, 1)
with self.assertRaises(ValueError):
- statistics._sqrt_frac(1, -1)
+ statistics._float_sqrt_of_frac(1, -1)
# Error handling for zero denominator matches that for Fraction(1, 0)
with self.assertRaises(ZeroDivisionError):
- statistics._sqrt_frac(1, 0)
+ statistics._float_sqrt_of_frac(1, 0)
# The result is well defined if both inputs are negative
- self.assertAlmostEqual(statistics._sqrt_frac(-2, -1), math.sqrt(2.0))
+ self.assertEqual(statistics._float_sqrt_of_frac(-2, -1), statistics._float_sqrt_of_frac(2, 1))
+
+ def test_decimal_sqrt_of_frac(self):
+ root: Decimal
+ numerator: int
+ denominator: int
+
+ for root, numerator, denominator in [
+ (Decimal('0.4481904599041192673635338663'), 200874688349065940678243576378, 1000000000000000000000000000000), # No adj
+ (Decimal('0.7924949131383786609961759598'), 628048187350206338833590574929, 1000000000000000000000000000000), # Adj up
+ (Decimal('0.8500554152289934068192208727'), 722594208960136395984391238251, 1000000000000000000000000000000), # Adj down
+ ]:
+ with decimal.localcontext(decimal.DefaultContext):
+ self.assertEqual(statistics._decimal_sqrt_of_frac(numerator, denominator), root)
+
+ # Confirm expected root with a quad precision decimal computation
+ with decimal.localcontext(decimal.DefaultContext) as ctx:
+ ctx.prec *= 4
+ high_prec_ratio = Decimal(numerator) / Decimal(denominator)
+ ctx.rounding = decimal.ROUND_05UP
+ high_prec_root = high_prec_ratio.sqrt()
+ with decimal.localcontext(decimal.DefaultContext):
+ target_root = +high_prec_root
+ self.assertEqual(root, target_root)
+
+ # Verify that corner cases and error handling match Decimal.sqrt()
+ self.assertEqual(statistics._decimal_sqrt_of_frac(0, 1), 0.0)
+ with self.assertRaises(decimal.InvalidOperation):
+ statistics._decimal_sqrt_of_frac(-1, 1)
+ with self.assertRaises(decimal.InvalidOperation):
+ statistics._decimal_sqrt_of_frac(1, -1)
+
+ # Error handling for zero denominator matches that for Fraction(1, 0)
+ with self.assertRaises(ZeroDivisionError):
+ statistics._decimal_sqrt_of_frac(1, 0)
+
+ # The result is well defined if both inputs are negative
+ self.assertEqual(statistics._decimal_sqrt_of_frac(-2, -1), statistics._decimal_sqrt_of_frac(2, 1))
class TestStdev(VarianceStdevMixin, NumericTestCase):
1
0