Mailman 3 May 2022 - Python-checkins

bpo-40514: Drop EXPERIMENTAL_ISOLATED_SUBINTERPRETERS (gh-93185)
by ericsnowcurrently May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/caa279d6fd5f151e57f891cd4f6ba51b53… commit: caa279d6fd5f151e57f891cd4f6ba51b532501c6 branch: main author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: ericsnowcurrently <ericsnowcurrently(a)gmail.com> date: 2022-05-27T17:38:01-06:00 summary: bpo-40514: Drop EXPERIMENTAL_ISOLATED_SUBINTERPRETERS (gh-93185) This was added for bpo-40514 (gh-84694) to test out a per-interpreter GIL. However, it has since proven unnecessary to keep the experiment in the repo. (It can be done as a branch in a fork like normal.) So here we are removing: * the configure option * the macro * the code enabled by the macro files: A Misc/NEWS.d/next/Core and Builtins/2022-05-25-12-30-12.gh-issue-84694.5sjy2w.rst M Doc/whatsnew/3.12.rst M Include/internal/pycore_ceval.h M Include/internal/pycore_interp.h M Include/internal/pycore_pystate.h M Include/internal/pycore_runtime.h M Modules/_xxsubinterpretersmodule.c M Modules/gcmodule.c M Objects/typeobject.c M Objects/unicodeobject.c M Python/ceval.c M Python/ceval_gil.h M Python/preconfig.c M Python/pylifecycle.c M Python/pystate.c M configure M configure.ac M pyconfig.h.in diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4493476cb0318..8a7309726ed2a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -115,6 +115,10 @@ Removed (Contributed by Erlend E. Aasland in :gh:`92548`) +* The ``--experimental-isolated-subinterpreters`` configure flag + (and corresponding ``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS``) + have been removed. + Porting to Python 3.12 ====================== diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 972eb926b3fa2..dafe31b216bf0 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -81,11 +81,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject* const* args, size_t argcount, PyObject *kwnames); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp); -#else extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); -#endif extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); extern void _PyEval_FiniGIL(PyInterpreterState *interp); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d55627908a28f..bcfcd88d84492 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -51,9 +51,6 @@ struct _ceval_state { /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; struct _pending_calls pending; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state gil; -#endif }; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index c4bc53c707fda..e2d7c5bee28b9 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -64,18 +64,10 @@ _Py_ThreadCanHandlePendingCalls(void) /* Variable and macro for in-line access to current thread and interpreter state */ -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -PyAPI_FUNC(PyThreadState*) _PyThreadState_GetTSS(void); -#endif - static inline PyThreadState* _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->gilstate.tstate_current); -#endif } /* Get the current Python thread state. @@ -90,11 +82,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) static inline PyThreadState* _PyThreadState_GET(void) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return _PyRuntimeState_GetThreadState(&_PyRuntime); -#endif } PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 18191c3771dfc..ae63ae74afa5f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,9 +23,7 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS struct _gil_runtime_state gil; -#endif }; /* GIL state */ diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-25-12-30-12.gh-issue-84694.5sjy2w.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-25-12-30-12.gh-issue-84694.5sjy2w.rst new file mode 100644 index 0000000000000..c062d28b29486 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-25-12-30-12.gh-issue-84694.5sjy2w.rst @@ -0,0 +1,2 @@ +The ``--experimental-isolated-subinterpreters`` configure option and +``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS`` macro have been removed. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index ee04e308fe554..eed9a8a1d85d5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1932,20 +1932,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, return -1; } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - // Switch to interpreter. - PyThreadState *new_tstate = PyInterpreterState_ThreadHead(interp); - PyThreadState *save1 = PyEval_SaveThread(); - - (void)PyThreadState_Swap(new_tstate); - - // Run the script. - _sharedexception *exc = NULL; - int result = _run_script(interp, codestr, shared, &exc); - - // Switch back. - PyEval_RestoreThread(save1); -#else // Switch to interpreter. PyThreadState *save_tstate = NULL; if (interp != PyInterpreterState_Get()) { @@ -1963,7 +1949,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, if (save_tstate != NULL) { PyThreadState_Swap(save_tstate); } -#endif // Propagate any exception out to the caller. if (exc != NULL) { diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 45dad8f0824df..3bda6e4bb8c02 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1195,14 +1195,6 @@ gc_collect_main(PyThreadState *tstate, int generation, assert(gcstate->garbage != NULL); assert(!_PyErr_Occurred(tstate)); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - if (tstate->interp->config._isolated_interpreter) { - // bpo-40533: The garbage collector must not be run on parallel on - // Python objects shared by multiple interpreters. - return 0; - } -#endif - if (gcstate->debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 37ac886f00446..2566d217a0f3b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -54,11 +54,6 @@ typedef struct PySlot_Offset { } PySlot_Offset; -/* bpo-40521: Interned strings are shared by all subinterpreters */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -# define INTERN_NAME_STRINGS -#endif - static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -4027,7 +4022,7 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) if (name == NULL) return -1; } -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Interned strings are shared by all subinterpreters */ if (!PyUnicode_CHECK_INTERNED(name)) { PyUnicode_InternInPlace(&name); if (!PyUnicode_CHECK_INTERNED(name)) { @@ -4037,7 +4032,6 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) return -1; } } -#endif } else { /* Will fail in _PyObject_GenericSetAttrWithDict. */ @@ -8474,17 +8468,11 @@ _PyTypes_InitSlotDefs(void) for (slotdef *p = slotdefs; p->name; p++) { /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Interned strings are shared by all subinterpreters */ p->name_strobj = PyUnicode_InternFromString(p->name); if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) { return _PyStatus_NO_MEMORY(); } -#else - p->name_strobj = PyUnicode_FromString(p->name); - if (!p->name_strobj) { - return _PyStatus_NO_MEMORY(); - } -#endif } slotdefs_initialized = 1; return _PyStatus_OK(); @@ -8509,24 +8497,17 @@ update_slot(PyTypeObject *type, PyObject *name) int offset; assert(PyUnicode_CheckExact(name)); -#ifdef INTERN_NAME_STRINGS assert(PyUnicode_CHECK_INTERNED(name)); -#endif assert(slotdefs_initialized); pp = ptrs; for (p = slotdefs; p->name; p++) { assert(PyUnicode_CheckExact(p->name_strobj)); assert(PyUnicode_CheckExact(name)); -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Using interned strings. */ if (p->name_strobj == name) { *pp++ = p; } -#else - if (p->name_strobj == name || _PyUnicode_EQ(p->name_strobj, name)) { - *pp++ = p; - } -#endif } *pp = NULL; for (pp = ptrs; *pp; pp++) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f712fbfa71d36..d0e4bcd920e40 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -190,11 +190,6 @@ extern "C" { # define OVERALLOCATE_FACTOR 4 #endif -/* bpo-40521: Interned strings are shared by all interpreters. */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -# define INTERNED_STRINGS -#endif - /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -203,9 +198,7 @@ extern "C" { Another way to look at this is that to say that the actual reference count of a string is: s->ob_refcnt + (s->state ? 2 : 0) */ -#ifdef INTERNED_STRINGS static PyObject *interned = NULL; -#endif /* Forward declaration */ static inline int @@ -1530,7 +1523,6 @@ unicode_dealloc(PyObject *unicode) } #endif -#ifdef INTERNED_STRINGS if (PyUnicode_CHECK_INTERNED(unicode)) { /* Revive the dead object temporarily. PyDict_DelItem() removes two references (key and value) which were ignored by @@ -1546,7 +1538,6 @@ unicode_dealloc(PyObject *unicode) assert(Py_REFCNT(unicode) == 1); Py_SET_REFCNT(unicode, 0); } -#endif if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyObject_Free(_PyUnicode_UTF8(unicode)); @@ -10568,13 +10559,11 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (PyUnicode_CHECK_INTERNED(left)) return 0; -#ifdef INTERNED_STRINGS assert(_PyUnicode_HASH(right_uni) != -1); Py_hash_t hash = _PyUnicode_HASH(left); if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { return 0; } -#endif return unicode_compare_eq(left, right_uni); } @@ -14645,7 +14634,6 @@ PyUnicode_InternInPlace(PyObject **p) return; } -#ifdef INTERNED_STRINGS if (interned == NULL) { interned = PyDict_New(); if (interned == NULL) { @@ -14671,11 +14659,6 @@ PyUnicode_InternInPlace(PyObject **p) this. */ Py_SET_REFCNT(s, Py_REFCNT(s) - 2); _PyUnicode_STATE(s).interned = 1; -#else - // PyDict expects that interned strings have their hash - // (PyASCIIObject.hash) already computed. - (void)unicode_hash(s); -#endif } // Function kept for the stable ABI. diff --git a/Python/ceval.c b/Python/ceval.c index 6158524999a84..fdbf0d4d6e99d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -352,21 +352,6 @@ _Py_FatalError_TstateNULL(const char *func) "(the current Python thread state is NULL)"); } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -int -_PyEval_ThreadsInitialized(PyInterpreterState *interp) -{ - return gil_created(&interp->ceval.gil); -} - -int -PyEval_ThreadsInitialized(void) -{ - // Fatal error if there is no current interpreter - PyInterpreterState *interp = PyInterpreterState_Get(); - return _PyEval_ThreadsInitialized(interp); -} -#else int _PyEval_ThreadsInitialized(_PyRuntimeState *runtime) { @@ -379,25 +364,18 @@ PyEval_ThreadsInitialized(void) _PyRuntimeState *runtime = &_PyRuntime; return _PyEval_ThreadsInitialized(runtime); } -#endif PyStatus _PyEval_InitGIL(PyThreadState *tstate) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return _PyStatus_OK(); } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; -#else struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; -#endif assert(!gil_created(gil)); PyThread_init_thread(); @@ -412,20 +390,14 @@ _PyEval_InitGIL(PyThreadState *tstate) void _PyEval_FiniGIL(PyInterpreterState *interp) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return; } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; -#endif if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -489,13 +461,9 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - (void)_PyThreadState_Swap(gilstate, tstate); -#else if (_PyThreadState_Swap(gilstate, tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } -#endif } void @@ -522,11 +490,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate) { _PyRuntimeState *runtime = tstate->interp->runtime; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; -#else struct _gil_runtime_state *gil = &runtime->ceval.gil; -#endif if (!gil_created(gil)) { return _PyStatus_OK(); } @@ -558,21 +522,12 @@ PyThreadState * PyEval_SaveThread(void) { _PyRuntimeState *runtime = &_PyRuntime; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThreadState *old_tstate = _PyThreadState_GET(); - PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, old_tstate); -#else PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); -#endif _Py_EnsureTstateNotNULL(tstate); struct _ceval_runtime_state *ceval = &runtime->ceval; struct _ceval_state *ceval2 = &tstate->interp->ceval; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - assert(gil_created(&ceval2->gil)); -#else assert(gil_created(&ceval->gil)); -#endif drop_gil(ceval, ceval2, tstate); return tstate; } @@ -836,9 +791,7 @@ Py_MakePendingCalls(void) void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS _gil_initialize(&ceval->gil); -#endif } void @@ -848,10 +801,6 @@ _PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock) assert(pending->lock == NULL); pending->lock = pending_lock; - -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - _gil_initialize(&ceval->gil); -#endif } void @@ -1267,13 +1216,9 @@ eval_frame_handle_pending(PyThreadState *tstate) take_gil(tstate); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - (void)_PyThreadState_Swap(&runtime->gilstate, tstate); -#else if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("orphan tstate"); } -#endif } /* Check for asynchronous exception. */ diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index 9b8b43253f04d..1b2dc7f8e1dc3 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -144,11 +144,7 @@ static void drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, PyThreadState *tstate) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &ceval2->gil; -#else struct _gil_runtime_state *gil = &ceval->gil; -#endif if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -232,11 +228,7 @@ take_gil(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; struct _ceval_runtime_state *ceval = &interp->runtime->ceval; struct _ceval_state *ceval2 = &interp->ceval; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &ceval2->gil; -#else struct _gil_runtime_state *gil = &ceval->gil; -#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -328,22 +320,12 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyInterpreterState *interp = PyInterpreterState_Get(); - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; -#endif gil->interval = microseconds; } unsigned long _PyEval_GetSwitchInterval() { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyInterpreterState *interp = PyInterpreterState_Get(); - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; -#endif return gil->interval; } diff --git a/Python/preconfig.c b/Python/preconfig.c index e7125cd2e3e0e..c30c75807a28a 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -294,17 +294,7 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) config->coerce_c_locale_warn = 0; config->dev_mode = -1; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - /* bpo-40512: pymalloc is not compatible with subinterpreters, - force usage of libc malloc() which is thread-safe. */ -#ifdef Py_DEBUG - config->allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; -#else - config->allocator = PYMEM_ALLOCATOR_MALLOC; -#endif -#else config->allocator = PYMEM_ALLOCATOR_NOT_SET; -#endif #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 42a058181dd69..31a358fe80de2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1981,12 +1981,10 @@ new_interpreter(PyThreadState **tstate_p, int isolated_subinterpreter) /* Copy the current interpreter config into the new interpreter */ const PyConfig *config; -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (save_tstate != NULL) { config = _PyInterpreterState_GetConfig(save_tstate->interp); } else -#endif { /* No current thread state, copy from the main interpreter */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); diff --git a/Python/pystate.c b/Python/pystate.c index 3e28a6ab69a98..df56c0530f05b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1165,14 +1165,6 @@ _PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -PyThreadState* -_PyThreadState_GetTSS(void) { - return PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey); -} -#endif - - PyThreadState * _PyThreadState_UncheckedGet(void) { @@ -1192,11 +1184,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThreadState *oldts = _PyThreadState_GetTSS(); -#else PyThreadState *oldts = _PyRuntimeGILState_GetThreadState(gilstate); -#endif _PyRuntimeGILState_SetThreadState(gilstate, newts); /* It should not be possible for more than one thread state @@ -1214,9 +1202,6 @@ _PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *new Py_FatalError("Invalid thread state for this thread"); errno = err; } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThread_tss_set(&gilstate->autoTSSkey, newts); #endif return oldts; } @@ -1665,9 +1650,7 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS assert(_PyEval_ThreadsInitialized(runtime)); -#endif assert(gilstate->autoInterpreterState); PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey); diff --git a/configure b/configure index 04f0c4e0e8842..de2983aee7c08 100755 --- a/configure +++ b/configure @@ -1061,7 +1061,6 @@ with_openssl with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes -with_experimental_isolated_subinterpreters enable_test_modules ' ac_precious_vars='build_alias @@ -1861,9 +1860,6 @@ Optional Packages: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 builtin hash modules, md5, sha1, sha256, sha512, sha3 (with shake), blake2 - --with-experimental-isolated-subinterpreters - better isolate subinterpreters, experimental build - mode (default is no) Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -22363,30 +22359,6 @@ fi fi -# --with-experimental-isolated-subinterpreters - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-experimental-isolated-subinterpreters" >&5 -$as_echo_n "checking for --with-experimental-isolated-subinterpreters... " >&6; } - -# Check whether --with-experimental-isolated-subinterpreters was given. -if test "${with_experimental_isolated_subinterpreters+set}" = set; then : - withval=$with_experimental_isolated_subinterpreters; -if test "$withval" != no -then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; }; - $as_echo "#define EXPERIMENTAL_ISOLATED_SUBINTERPRETERS 1" >>confdefs.h - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; }; -fi -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --disable-test-modules" >&5 diff --git a/configure.ac b/configure.ac index 9b5926d79d44e..e925bc0c8df1a 100644 --- a/configure.ac +++ b/configure.ac @@ -6540,23 +6540,6 @@ AS_VAR_IF([with_builtin_blake2], [yes], [ ], [have_libb2=no]) ]) -# --with-experimental-isolated-subinterpreters -AH_TEMPLATE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS, - [Better isolate subinterpreters, experimental build mode.]) -AC_MSG_CHECKING(for --with-experimental-isolated-subinterpreters) -AC_ARG_WITH(experimental-isolated-subinterpreters, - AS_HELP_STRING([--with-experimental-isolated-subinterpreters], - [better isolate subinterpreters, experimental build mode (default is no)]), -[ -if test "$withval" != no -then - AC_MSG_RESULT(yes); - AC_DEFINE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS) -else - AC_MSG_RESULT(no); -fi], -[AC_MSG_RESULT(no)]) - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. AC_MSG_CHECKING([for --disable-test-modules]) diff --git a/pyconfig.h.in b/pyconfig.h.in index a3bbbb894fd56..1fba0c68aae4b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -44,9 +44,6 @@ /* Define if --enable-ipv6 is specified */ #undef ENABLE_IPV6 -/* Better isolate subinterpreters, experimental build mode. */ -#undef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - /* Define to 1 if your system stores words within floats with the most significant word first */ #undef FLOAT_WORDS_BIGENDIAN

1 0

gh-93297: Make asyncio task groups prevent child tasks from being GCed (GH-93299) (#93305)
by 1st1 May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/cf63b80bc482ef971ecb6d3ed9a1dc4a93… commit: cf63b80bc482ef971ecb6d3ed9a1dc4a93d73744 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: 1st1 <yury(a)edgedb.com> date: 2022-05-27T15:46:57-07:00 summary: gh-93297: Make asyncio task groups prevent child tasks from being GCed (GH-93299) (#93305) (cherry picked from commit e6a57678cafe18ca132ee9510252168fcc392a8d) Co-authored-by: Yury Selivanov <yury(a)edgedb.com> Co-authored-by: Yury Selivanov <yury(a)edgedb.com> files: A Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst M Lib/asyncio/taskgroups.py diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 6af21f3a15d93..9e0610deed281 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -3,8 +3,6 @@ __all__ = ["TaskGroup"] -import weakref - from . import events from . import exceptions from . import tasks @@ -19,8 +17,7 @@ def __init__(self): self._loop = None self._parent_task = None self._parent_cancel_requested = False - self._tasks = weakref.WeakSet() - self._unfinished_tasks = 0 + self._tasks = set() self._errors = [] self._base_error = None self._on_completed_fut = None @@ -29,8 +26,6 @@ def __repr__(self): info = [''] if self._tasks: info.append(f'tasks={len(self._tasks)}') - if self._unfinished_tasks: - info.append(f'unfinished={self._unfinished_tasks}') if self._errors: info.append(f'errors={len(self._errors)}') if self._aborting: @@ -93,7 +88,7 @@ async def __aexit__(self, et, exc, tb): # can be cancelled multiple times if our parent task # is being cancelled repeatedly (or even once, when # our own cancellation is already in progress) - while self._unfinished_tasks: + while self._tasks: if self._on_completed_fut is None: self._on_completed_fut = self._loop.create_future() @@ -114,7 +109,7 @@ async def __aexit__(self, et, exc, tb): self._on_completed_fut = None - assert self._unfinished_tasks == 0 + assert not self._tasks if self._base_error is not None: raise self._base_error @@ -141,7 +136,7 @@ async def __aexit__(self, et, exc, tb): def create_task(self, coro, *, name=None, context=None): if not self._entered: raise RuntimeError(f"TaskGroup {self!r} has not been entered") - if self._exiting and self._unfinished_tasks == 0: + if self._exiting and not self._tasks: raise RuntimeError(f"TaskGroup {self!r} is finished") if context is None: task = self._loop.create_task(coro) @@ -149,7 +144,6 @@ def create_task(self, coro, *, name=None, context=None): task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) task.add_done_callback(self._on_task_done) - self._unfinished_tasks += 1 self._tasks.add(task) return task @@ -169,10 +163,9 @@ def _abort(self): t.cancel() def _on_task_done(self, task): - self._unfinished_tasks -= 1 - assert self._unfinished_tasks >= 0 + self._tasks.discard(task) - if self._on_completed_fut is not None and not self._unfinished_tasks: + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst b/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst new file mode 100644 index 0000000000000..a8e4cd93d3047 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst @@ -0,0 +1 @@ +Make asyncio task groups prevent child tasks from being GCed

1 0

gh-93250: [Enum] Change IntEnum boundary to KEEP for backwards compatibility (GH-93302) (GH-93304)
by ethanfurman May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/647426d4fa272da33269a9c183f3949a34… commit: 647426d4fa272da33269a9c183f3949a3477202d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2022-05-27T15:43:13-07:00 summary: gh-93250: [Enum] Change IntEnum boundary to KEEP for backwards compatibility (GH-93302) (GH-93304) In previous versions of Python if an IntEnum member was combined with another integer type value using a bit-wise operation, the resulting value would still be the IntEnum type. This change restores that behavior. (cherry picked from commit 70cfe56cafb2b549983f63d5d1a54654fe63c15c) Co-authored-by: Ethan Furman <ethan(a)stoneleaf.us> files: M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index 4a21226f0193f..fe521b1018d48 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1575,7 +1575,7 @@ def __invert__(self): __rxor__ = __xor__ -class IntFlag(int, ReprEnum, Flag, boundary=EJECT): +class IntFlag(int, ReprEnum, Flag, boundary=KEEP): """ Support for integer-based Flags """ diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index c9ed08397d77c..56cebfea3f1ba 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3349,7 +3349,10 @@ def test_invert(self): self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) def test_boundary(self): - self.assertIs(enum.IntFlag._boundary_, EJECT) + self.assertIs(enum.IntFlag._boundary_, KEEP) + class Simple(IntFlag, boundary=KEEP): + SINGLE = 1 + # class Iron(IntFlag, boundary=STRICT): ONE = 1 TWO = 2 @@ -3368,7 +3371,6 @@ class Space(IntFlag, boundary=EJECT): EIGHT = 8 self.assertIs(Space._boundary_, EJECT) # - # class Bizarre(IntFlag, boundary=KEEP): b = 3 c = 4 @@ -3385,6 +3387,12 @@ class Bizarre(IntFlag, boundary=KEEP): self.assertEqual(list(Bizarre), [Bizarre.c]) self.assertIs(Bizarre(3), Bizarre.b) self.assertIs(Bizarre(6), Bizarre.d) + # + simple = Simple.SINGLE | Iron.TWO + self.assertEqual(simple, 3) + self.assertIsInstance(simple, Simple) + self.assertEqual(repr(simple), '<Simple.SINGLE|<Iron.TWO: 2>: 3>') + self.assertEqual(str(simple), '3') def test_iter(self): Color = self.Color

1 0

gh-91243: Document Required and NotRequired (GH-93173)
by miss-islington May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/3f7abff97bf6472f35d23a00656e88d77c… commit: 3f7abff97bf6472f35d23a00656e88d77c93feea branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-27T15:25:50-07:00 summary: gh-91243: Document Required and NotRequired (GH-93173) Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> (cherry picked from commit 711eda7dec1f62ecb2a87b274b5f6bfc2b6fbef4) Co-authored-by: David Foster <david(a)dafoster.net> files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 7ddc84ac5e451..c7c5536a64fb4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -78,6 +78,8 @@ annotations. These include: *Introducing* :data:`TypeVarTuple` * :pep:`647`: User-Defined Type Guards *Introducing* :data:`TypeGuard` +* :pep:`655`: Marking individual TypedDict items as required or potentially-missing + *Introducing* :data:`Required` and :data:`NotRequired` * :pep:`673`: Self type *Introducing* :data:`Self` * :pep:`675`: Arbitrary Literal String Type @@ -1022,6 +1024,18 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.8 +.. data:: Required + +.. data:: NotRequired + + Special typing constructs that mark individual keys of a :class:`TypedDict` + as either required or non-required respectively. + + For more information, see :class:`TypedDict` and + :pep:`655` ("Marking individual TypedDict items as required or potentially-missing"). + + .. versionadded:: 3.11 + .. data:: Annotated A type, introduced in :pep:`593` (``Flexible function and variable @@ -1706,8 +1720,21 @@ These are not used in annotations. They are building blocks for declaring types. Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) By default, all keys must be present in a ``TypedDict``. It is possible to - override this by specifying totality. - Usage:: + mark individual keys as non-required using :data:`NotRequired`:: + + class Point2D(TypedDict): + x: int + y: int + label: NotRequired[str] + + # Alternative syntax + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) + + This means that a ``Point2D`` ``TypedDict`` can have the ``label`` + key omitted. + + It is also possible to mark all keys as non-required by default + by specifying a totality of ``False``:: class Point2D(TypedDict, total=False): x: int @@ -1721,6 +1748,21 @@ These are not used in annotations. They are building blocks for declaring types. ``True`` as the value of the ``total`` argument. ``True`` is the default, and makes all items defined in the class body required. + Individual keys of a ``total=False`` ``TypedDict`` can be marked as + required using :data:`Required`:: + + class Point2D(TypedDict, total=False): + x: Required[int] + y: Required[int] + label: str + + # Alternative syntax + Point2D = TypedDict('Point2D', { + 'x': Required[int], + 'y': Required[int], + 'label': str + }, total=False) + It is possible for a ``TypedDict`` type to inherit from one or more other ``TypedDict`` types using the class-based syntax. Usage:: @@ -1785,11 +1827,16 @@ These are not used in annotations. They are building blocks for declaring types. ``Point2D.__required_keys__`` and ``Point2D.__optional_keys__`` return :class:`frozenset` objects containing required and non-required keys, respectively. - Currently the only way to declare both required and non-required keys in the - same ``TypedDict`` is mixed inheritance, declaring a ``TypedDict`` with one value - for the ``total`` argument and then inheriting it from another ``TypedDict`` with - a different value for ``total``. - Usage:: + + Keys marked with :data:`Required` will always appear in ``__required_keys__`` + and keys marked with :data:`NotRequired` will always appear in ``__optional_keys__``. + + For backwards compatibility with Python 3.10 and below, + it is also possible to use inheritance to declare both required and + non-required keys in the same ``TypedDict`` . This is done by declaring a + ``TypedDict`` with one value for the ``total`` argument and then + inheriting from it in another ``TypedDict`` with a different value for + ``total``:: >>> class Point2D(TypedDict, total=False): ... x: int @@ -1807,6 +1854,10 @@ These are not used in annotations. They are building blocks for declaring types. .. versionadded:: 3.8 + .. versionchanged:: 3.11 + Added support for marking individual keys as :data:`Required` or :data:`NotRequired`. + See :pep:`655`. + .. versionchanged:: 3.11 Added support for generic ``TypedDict``\ s.

1 0

gh-93297: Make asyncio task groups prevent child tasks from being GCed (#93299)
by 1st1 May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/e6a57678cafe18ca132ee9510252168fcc… commit: e6a57678cafe18ca132ee9510252168fcc392a8d branch: main author: Yury Selivanov <yury(a)edgedb.com> committer: 1st1 <yury(a)edgedb.com> date: 2022-05-27T15:20:21-07:00 summary: gh-93297: Make asyncio task groups prevent child tasks from being GCed (#93299) files: A Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst M Lib/asyncio/taskgroups.py diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 6af21f3a15d93..9e0610deed281 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -3,8 +3,6 @@ __all__ = ["TaskGroup"] -import weakref - from . import events from . import exceptions from . import tasks @@ -19,8 +17,7 @@ def __init__(self): self._loop = None self._parent_task = None self._parent_cancel_requested = False - self._tasks = weakref.WeakSet() - self._unfinished_tasks = 0 + self._tasks = set() self._errors = [] self._base_error = None self._on_completed_fut = None @@ -29,8 +26,6 @@ def __repr__(self): info = [''] if self._tasks: info.append(f'tasks={len(self._tasks)}') - if self._unfinished_tasks: - info.append(f'unfinished={self._unfinished_tasks}') if self._errors: info.append(f'errors={len(self._errors)}') if self._aborting: @@ -93,7 +88,7 @@ async def __aexit__(self, et, exc, tb): # can be cancelled multiple times if our parent task # is being cancelled repeatedly (or even once, when # our own cancellation is already in progress) - while self._unfinished_tasks: + while self._tasks: if self._on_completed_fut is None: self._on_completed_fut = self._loop.create_future() @@ -114,7 +109,7 @@ async def __aexit__(self, et, exc, tb): self._on_completed_fut = None - assert self._unfinished_tasks == 0 + assert not self._tasks if self._base_error is not None: raise self._base_error @@ -141,7 +136,7 @@ async def __aexit__(self, et, exc, tb): def create_task(self, coro, *, name=None, context=None): if not self._entered: raise RuntimeError(f"TaskGroup {self!r} has not been entered") - if self._exiting and self._unfinished_tasks == 0: + if self._exiting and not self._tasks: raise RuntimeError(f"TaskGroup {self!r} is finished") if context is None: task = self._loop.create_task(coro) @@ -149,7 +144,6 @@ def create_task(self, coro, *, name=None, context=None): task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) task.add_done_callback(self._on_task_done) - self._unfinished_tasks += 1 self._tasks.add(task) return task @@ -169,10 +163,9 @@ def _abort(self): t.cancel() def _on_task_done(self, task): - self._unfinished_tasks -= 1 - assert self._unfinished_tasks >= 0 + self._tasks.discard(task) - if self._on_completed_fut is not None and not self._unfinished_tasks: + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst b/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst new file mode 100644 index 0000000000000..a8e4cd93d3047 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-27-13-18-18.gh-issue-93297.e2zuHz.rst @@ -0,0 +1 @@ +Make asyncio task groups prevent child tasks from being GCed

1 0

gh-93250: [Enum] Change IntEnum boundary to KEEP for backwards compatibility (GH-93302)
by ethanfurman May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/70cfe56cafb2b549983f63d5d1a54654fe… commit: 70cfe56cafb2b549983f63d5d1a54654fe63c15c branch: main author: Ethan Furman <ethan(a)stoneleaf.us> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2022-05-27T15:14:28-07:00 summary: gh-93250: [Enum] Change IntEnum boundary to KEEP for backwards compatibility (GH-93302) In previous versions of Python if an IntEnum member was combined with another integer type value using a bit-wise operation, the resulting value would still be the IntEnum type. This change restores that behavior. files: M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index 0b97d3d8a68ef..4261b117d9db2 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1572,7 +1572,7 @@ def __invert__(self): __rxor__ = __xor__ -class IntFlag(int, ReprEnum, Flag, boundary=EJECT): +class IntFlag(int, ReprEnum, Flag, boundary=KEEP): """ Support for integer-based Flags """ diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 44a3912630c32..25b3e2d818929 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -3353,7 +3353,10 @@ def test_invert(self): self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) def test_boundary(self): - self.assertIs(enum.IntFlag._boundary_, EJECT) + self.assertIs(enum.IntFlag._boundary_, KEEP) + class Simple(IntFlag, boundary=KEEP): + SINGLE = 1 + # class Iron(IntFlag, boundary=STRICT): ONE = 1 TWO = 2 @@ -3372,7 +3375,6 @@ class Space(IntFlag, boundary=EJECT): EIGHT = 8 self.assertIs(Space._boundary_, EJECT) # - # class Bizarre(IntFlag, boundary=KEEP): b = 3 c = 4 @@ -3389,6 +3391,12 @@ class Bizarre(IntFlag, boundary=KEEP): self.assertEqual(list(Bizarre), [Bizarre.c]) self.assertIs(Bizarre(3), Bizarre.b) self.assertIs(Bizarre(6), Bizarre.d) + # + simple = Simple.SINGLE | Iron.TWO + self.assertEqual(simple, 3) + self.assertIsInstance(simple, Simple) + self.assertEqual(repr(simple), '<Simple.SINGLE|<Iron.TWO: 2>: 3>') + self.assertEqual(str(simple), '3') def test_iter(self): Color = self.Color

1 0

gh-91243: Document Required and NotRequired (#93173)
by JelleZijlstra May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/711eda7dec1f62ecb2a87b274b5f6bfc2b… commit: 711eda7dec1f62ecb2a87b274b5f6bfc2b6fbef4 branch: main author: David Foster <david(a)dafoster.net> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2022-05-27T15:11:45-07:00 summary: gh-91243: Document Required and NotRequired (#93173) Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 7ddc84ac5e451..c7c5536a64fb4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -78,6 +78,8 @@ annotations. These include: *Introducing* :data:`TypeVarTuple` * :pep:`647`: User-Defined Type Guards *Introducing* :data:`TypeGuard` +* :pep:`655`: Marking individual TypedDict items as required or potentially-missing + *Introducing* :data:`Required` and :data:`NotRequired` * :pep:`673`: Self type *Introducing* :data:`Self` * :pep:`675`: Arbitrary Literal String Type @@ -1022,6 +1024,18 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.8 +.. data:: Required + +.. data:: NotRequired + + Special typing constructs that mark individual keys of a :class:`TypedDict` + as either required or non-required respectively. + + For more information, see :class:`TypedDict` and + :pep:`655` ("Marking individual TypedDict items as required or potentially-missing"). + + .. versionadded:: 3.11 + .. data:: Annotated A type, introduced in :pep:`593` (``Flexible function and variable @@ -1706,8 +1720,21 @@ These are not used in annotations. They are building blocks for declaring types. Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) By default, all keys must be present in a ``TypedDict``. It is possible to - override this by specifying totality. - Usage:: + mark individual keys as non-required using :data:`NotRequired`:: + + class Point2D(TypedDict): + x: int + y: int + label: NotRequired[str] + + # Alternative syntax + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) + + This means that a ``Point2D`` ``TypedDict`` can have the ``label`` + key omitted. + + It is also possible to mark all keys as non-required by default + by specifying a totality of ``False``:: class Point2D(TypedDict, total=False): x: int @@ -1721,6 +1748,21 @@ These are not used in annotations. They are building blocks for declaring types. ``True`` as the value of the ``total`` argument. ``True`` is the default, and makes all items defined in the class body required. + Individual keys of a ``total=False`` ``TypedDict`` can be marked as + required using :data:`Required`:: + + class Point2D(TypedDict, total=False): + x: Required[int] + y: Required[int] + label: str + + # Alternative syntax + Point2D = TypedDict('Point2D', { + 'x': Required[int], + 'y': Required[int], + 'label': str + }, total=False) + It is possible for a ``TypedDict`` type to inherit from one or more other ``TypedDict`` types using the class-based syntax. Usage:: @@ -1785,11 +1827,16 @@ These are not used in annotations. They are building blocks for declaring types. ``Point2D.__required_keys__`` and ``Point2D.__optional_keys__`` return :class:`frozenset` objects containing required and non-required keys, respectively. - Currently the only way to declare both required and non-required keys in the - same ``TypedDict`` is mixed inheritance, declaring a ``TypedDict`` with one value - for the ``total`` argument and then inheriting it from another ``TypedDict`` with - a different value for ``total``. - Usage:: + + Keys marked with :data:`Required` will always appear in ``__required_keys__`` + and keys marked with :data:`NotRequired` will always appear in ``__optional_keys__``. + + For backwards compatibility with Python 3.10 and below, + it is also possible to use inheritance to declare both required and + non-required keys in the same ``TypedDict`` . This is done by declaring a + ``TypedDict`` with one value for the ``total`` argument and then + inheriting from it in another ``TypedDict`` with a different value for + ``total``:: >>> class Point2D(TypedDict, total=False): ... x: int @@ -1807,6 +1854,10 @@ These are not used in annotations. They are building blocks for declaring types. .. versionadded:: 3.8 + .. versionchanged:: 3.11 + Added support for marking individual keys as :data:`Required` or :data:`NotRequired`. + See :pep:`655`. + .. versionchanged:: 3.11 Added support for generic ``TypedDict``\ s.

1 0

gh-77024: test.support: Improve documentation (GH-92513)
by miss-islington May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/9912b3d989b0cb442e9f9d55fbdd30d555… commit: 9912b3d989b0cb442e9f9d55fbdd30d55591e2fc branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-05-27T12:25:21-07:00 summary: gh-77024: test.support: Improve documentation (GH-92513) This is a rework of GH-5774 on current main. I was a bit more conservative in making changes than the original PR. See @csabella's comments on issue GH-77024 and the discussion on GH-5774 for explanations of several of the changes. Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> (cherry picked from commit 8995177030c8b41885ad92b260279b7e622ecaea) Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> files: M Doc/library/test.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index a8dc35476fc9b..07c3339661ad5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -359,13 +359,19 @@ The :mod:`test.support` module defines the following constants: .. data:: MISSING_C_DOCSTRINGS - Return ``True`` if running on CPython, not on Windows, and configuration - not set with ``WITH_DOC_STRINGS``. + Set to ``True`` if Python is built without docstrings (the + :c:macro:`WITH_DOC_STRINGS` macro is not defined). + See the :option:`configure --without-doc-strings <--without-doc-strings>` option. + + See also the :data:`HAVE_DOCSTRINGS` variable. .. data:: HAVE_DOCSTRINGS - Check for presence of docstrings. + Set to ``True`` if function docstrings are available. + See the :option:`python -OO <-O>` option, which strips docstrings of functions implemented in Python. + + See also the :data:`MISSING_C_DOCSTRINGS` variable. .. data:: TEST_HTTP_URL @@ -423,11 +429,6 @@ The :mod:`test.support` module defines the following functions: Used when tests are executed by :mod:`test.regrtest`. -.. function:: system_must_validate_cert(f) - - Raise :exc:`unittest.SkipTest` on TLS certification validation failures. - - .. function:: sortdict(dict) Return a repr of *dict* with keys sorted. @@ -445,12 +446,12 @@ The :mod:`test.support` module defines the following functions: .. function:: match_test(test) - Match *test* to patterns set in :func:`set_match_tests`. + Determine whether *test* matches the patterns set in :func:`set_match_tests`. -.. function:: set_match_tests(patterns) +.. function:: set_match_tests(accept_patterns=None, ignore_patterns=None) - Define match test with regular expression *patterns*. + Define match patterns on test filenames and test method names for filtering tests. .. function:: run_unittest(*classes) @@ -490,7 +491,9 @@ The :mod:`test.support` module defines the following functions: .. function:: check_impl_detail(**guards) Use this check to guard CPython's implementation-specific tests or to - run them only on the implementations guarded by the arguments:: + run them only on the implementations guarded by the arguments. This + function returns ``True`` or ``False`` depending on the host platform. + Example usage:: check_impl_detail() # Only on CPython (default). check_impl_detail(jython=True) # Only on Jython. @@ -509,7 +512,7 @@ The :mod:`test.support` module defines the following functions: time the regrtest began. -.. function:: get_original_stdout +.. function:: get_original_stdout() Return the original stdout set by :func:`record_original_stdout` or ``sys.stdout`` if it's not set. @@ -554,7 +557,7 @@ The :mod:`test.support` module defines the following functions: .. function:: disable_faulthandler() - A context manager that replaces ``sys.stderr`` with ``sys.__stderr__``. + A context manager that temporary disables :mod:`faulthandler`. .. function:: gc_collect() @@ -567,8 +570,8 @@ The :mod:`test.support` module defines the following functions: .. function:: disable_gc() - A context manager that disables the garbage collector upon entry and - reenables it upon exit. + A context manager that disables the garbage collector on entry. On + exit, the garbage collector is restored to its prior state. .. function:: swap_attr(obj, attr, new_val) @@ -633,14 +636,14 @@ The :mod:`test.support` module defines the following functions: .. function:: calcobjsize(fmt) - Return :func:`struct.calcsize` for ``nP{fmt}0n`` or, if ``gettotalrefcount`` - exists, ``2PnP{fmt}0P``. + Return the size of the :c:type:`PyObject` whose structure members are + defined by *fmt*. The returned value includes the size of the Python object header and alignment. .. function:: calcvobjsize(fmt) - Return :func:`struct.calcsize` for ``nPn{fmt}0n`` or, if ``gettotalrefcount`` - exists, ``2PnPn{fmt}0P``. + Return the size of the :c:type:`PyVarObject` whose structure members are + defined by *fmt*. The returned value includes the size of the Python object header and alignment. .. function:: checksizeof(test, o, size) @@ -656,6 +659,11 @@ The :mod:`test.support` module defines the following functions: have an associated comment identifying the relevant tracker issue. +.. function:: system_must_validate_cert(f) + + A decorator that skips the decorated test on TLS certification validation failures. + + .. decorator:: run_with_locale(catstr, *locales) A decorator for running a function in a different locale, correctly @@ -673,19 +681,19 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_freebsd_version(*min_version) Decorator for the minimum version when running test on FreeBSD. If the - FreeBSD version is less than the minimum, raise :exc:`unittest.SkipTest`. + FreeBSD version is less than the minimum, the test is skipped. .. decorator:: requires_linux_version(*min_version) Decorator for the minimum version when running test on Linux. If the - Linux version is less than the minimum, raise :exc:`unittest.SkipTest`. + Linux version is less than the minimum, the test is skipped. .. decorator:: requires_mac_version(*min_version) Decorator for the minimum version when running test on macOS. If the - macOS version is less than the minimum, raise :exc:`unittest.SkipTest`. + macOS version is less than the minimum, the test is skipped. .. decorator:: requires_IEEE_754 @@ -723,7 +731,7 @@ The :mod:`test.support` module defines the following functions: Decorator for only running the test if :data:`HAVE_DOCSTRINGS`. -.. decorator:: cpython_only(test) +.. decorator:: cpython_only Decorator for tests only applicable to CPython. @@ -734,12 +742,12 @@ The :mod:`test.support` module defines the following functions: returns ``False``, then uses *msg* as the reason for skipping the test. -.. decorator:: no_tracing(func) +.. decorator:: no_tracing Decorator to temporarily turn off tracing for the duration of the test. -.. decorator:: refcount_test(test) +.. decorator:: refcount_test Decorator for tests which involve reference counting. The decorator does not run the test if it is not run by CPython. Any trace function is unset @@ -762,10 +770,9 @@ The :mod:`test.support` module defines the following functions: means the test doesn't support dummy runs when ``-M`` is not specified. -.. decorator:: bigaddrspacetest(f) +.. decorator:: bigaddrspacetest - Decorator for tests that fill the address space. *f* is the function to - wrap. + Decorator for tests that fill the address space. .. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) @@ -867,7 +874,7 @@ The :mod:`test.support` module defines the following functions: .. function:: check_free_after_iterating(test, iter, cls, args=()) - Assert that *iter* is deallocated after iterating. + Assert instances of *cls* are deallocated after iterating. .. function:: missing_compiler_executable(cmd_names=[]) @@ -958,6 +965,16 @@ The :mod:`test.support` module defines the following classes: Class to save and restore signal handlers registered by the Python signal handler. + .. method:: save(self) + + Save the signal handlers to a dictionary mapping signal numbers to the + current signal handler. + + .. method:: restore(self) + + Set the signal numbers from the :meth:`save` dictionary to the saved + handler. + .. class:: Matcher() @@ -1101,11 +1118,11 @@ script execution tests. variables *env_vars* succeeds (``rc == 0``) and return a ``(return code, stdout, stderr)`` tuple. - If the ``__cleanenv`` keyword is set, *env_vars* is used as a fresh + If the *__cleanenv* keyword-only parameter is set, *env_vars* is used as a fresh environment. Python is started in isolated mode (command line option ``-I``), - except if the ``__isolated`` keyword is set to ``False``. + except if the *__isolated* keyword-only parameter is set to ``False``. .. versionchanged:: 3.9 The function no longer strips whitespaces from *stderr*. @@ -1216,15 +1233,17 @@ The :mod:`test.support.threading_helper` module provides support for threading t is still alive after *timeout* seconds. -.. decorator:: reap_threads(func) +.. decorator:: reap_threads Decorator to ensure the threads are cleaned up even if the test fails. .. function:: start_threads(threads, unlock=None) - Context manager to start *threads*. It attempts to join the threads upon - exit. + Context manager to start *threads*, which is a sequence of threads. + *unlock* is a function called after the threads are started, even if an + exception was raised; an example would be :meth:`threading.Event.set`. + ``start_threads`` will attempt to join the started threads upon exit. .. function:: threading_cleanup(*original_values) @@ -1306,7 +1325,10 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. data:: TESTFN_NONASCII - Set to a filename containing the :data:`FS_NONASCII` character. + Set to a filename containing the :data:`FS_NONASCII` character, if it exists. + This guarantees that if the filename exists, it can be encoded and decoded + with the default filesystem encoding. This allows tests that require a + non-ASCII filename to be easily skipped on platforms where they can't work. .. data:: TESTFN_UNENCODABLE @@ -1404,13 +1426,16 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: rmdir(filename) Call :func:`os.rmdir` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence of the file. + wrapped with a wait loop that checks for the existence of the file, + which is needed due to antivirus programs that can hold files open and prevent + deletion. .. function:: rmtree(path) Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and - :func:`os.rmdir` to remove a path and its contents. On Windows platforms, + :func:`os.rmdir` to remove a path and its contents. As with :func:`rmdir`, + on Windows platforms this is wrapped with a wait loop that checks for the existence of the files. @@ -1457,7 +1482,8 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: unlink(filename) - Call :func:`os.unlink` on *filename*. On Windows platforms, this is + Call :func:`os.unlink` on *filename*. As with :func:`rmdir`, + on Windows platforms, this is wrapped with a wait loop that checks for the existence of the file. @@ -1514,7 +1540,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. .. versionadded:: 3.1 -.. function:: import_module(name, deprecated=False, *, required_on()) +.. function:: import_module(name, deprecated=False, *, required_on=()) This function imports and returns the named module. Unlike a normal import, this function raises :exc:`unittest.SkipTest` if the module @@ -1556,7 +1582,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. A context manager to force import to return a new module reference. This is useful for testing module-level behaviors, such as the emission of a - DeprecationWarning on import. Example usage:: + :exc:`DeprecationWarning` on import. Example usage:: with CleanImport('foo'): importlib.import_module('foo') # New reference. @@ -1564,7 +1590,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. .. class:: DirsOnSysPath(*paths) - A context manager to temporarily add directories to sys.path. + A context manager to temporarily add directories to :data:`sys.path`. This makes a copy of :data:`sys.path`, appends any directories given as positional arguments, then reverts :data:`sys.path` to the copied

1 0

bpo-46951: Order contents of zipapps (GH-31713)
by serhiy-storchaka May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/47e68d451586541259ffb6ae29e948d4ea… commit: 47e68d451586541259ffb6ae29e948d4ea67c059 branch: main author: hfinucane <h.finucane+gh(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2022-05-27T19:04:29+03:00 summary: bpo-46951: Order contents of zipapps (GH-31713) So that builds are more reproducible. files: A Misc/NEWS.d/next/Library/2022-03-08-04-46-44.bpo-46951.SWAz97.rst M Lib/test/test_zipapp.py M Lib/zipapp.py diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index 69f2e55d56384..d135c68865812 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -54,6 +54,22 @@ def test_create_archive_with_subdirs(self): self.assertIn('foo/', z.namelist()) self.assertIn('bar/', z.namelist()) + def test_create_sorted_archive(self): + # Test that zipapps order their files by name + source = self.tmpdir / 'source' + source.mkdir() + (source / 'zed.py').touch() + (source / 'bin').mkdir() + (source / 'bin' / 'qux').touch() + (source / 'bin' / 'baz').touch() + (source / '__main__.py').touch() + target = io.BytesIO() + zipapp.create_archive(str(source), target) + target.seek(0) + with zipfile.ZipFile(target, 'r') as zf: + self.assertEqual(zf.namelist(), + ["__main__.py", "bin/", "bin/baz", "bin/qux", "zed.py"]) + def test_create_archive_with_filter(self): # Test packing a directory and using filter to specify # which files to include. diff --git a/Lib/zipapp.py b/Lib/zipapp.py index ce77632516c64..d8ebfcb6c73b0 100644 --- a/Lib/zipapp.py +++ b/Lib/zipapp.py @@ -136,7 +136,7 @@ def create_archive(source, target=None, interpreter=None, main=None, compression = (zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED) with zipfile.ZipFile(fd, 'w', compression=compression) as z: - for child in source.rglob('*'): + for child in sorted(source.rglob('*')): arcname = child.relative_to(source) if filter is None or filter(arcname): z.write(child, arcname.as_posix()) diff --git a/Misc/NEWS.d/next/Library/2022-03-08-04-46-44.bpo-46951.SWAz97.rst b/Misc/NEWS.d/next/Library/2022-03-08-04-46-44.bpo-46951.SWAz97.rst new file mode 100644 index 0000000000000..cd7601aa8c4de --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-08-04-46-44.bpo-46951.SWAz97.rst @@ -0,0 +1 @@ +Order the contents of zipapp archives, to make builds more reproducible.

1 0

GH-90230: Add stats to breakdown the origin of calls to `PyEval_EvalFrame` (GH-93284)
by markshannon May 27, 2022

May 27, 2022

https://github.com/python/cpython/commit/bbcf42449e13c0b62f145cd49d12674ef3… commit: bbcf42449e13c0b62f145cd49d12674ef3d5bf64 branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2022-05-27T16:31:41+01:00 summary: GH-90230: Add stats to breakdown the origin of calls to `PyEval_EvalFrame` (GH-93284) files: M Include/internal/pycore_call.h M Include/internal/pycore_ceval.h M Include/internal/pycore_code.h M Include/pystats.h M Modules/_asynciomodule.c M Objects/call.c M Objects/descrobject.c M Objects/genobject.c M Objects/typeobject.c M Python/bltinmodule.c M Python/ceval.c M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 3ccacfa0b8b03..55378e3dfebf2 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -103,6 +103,7 @@ _PyObject_CallNoArgsTstate(PyThreadState *tstate, PyObject *func) { // Private static inline function variant of public PyObject_CallNoArgs() static inline PyObject * _PyObject_CallNoArgs(PyObject *func) { + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); PyThreadState *tstate = _PyThreadState_GET(); return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } @@ -111,6 +112,7 @@ _PyObject_CallNoArgs(PyObject *func) { static inline PyObject * _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) { + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 3efd6beb035a1..972eb926b3fa2 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -68,6 +68,7 @@ extern PyObject* _PyEval_BuiltinsFromGlobals( static inline PyObject* _PyEval_EvalFrame(PyThreadState *tstate, struct _PyInterpreterFrame *frame, int throwflag) { + EVAL_CALL_STAT_INC(EVAL_CALL_TOTAL); if (tstate->interp->eval_frame == NULL) { return _PyEval_EvalFrameDefault(tstate, frame, throwflag); } diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 3d19843274112..2cfa319f2d290 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -265,6 +265,9 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co); #define OBJECT_STAT_INC(name) _py_stats.object_stats.name++ #define OBJECT_STAT_INC_COND(name, cond) \ do { if (cond) _py_stats.object_stats.name++; } while (0) +#define EVAL_CALL_STAT_INC(name) _py_stats.call_stats.eval_calls[name]++ +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ + do { if (PyFunction_Check(callable)) _py_stats.call_stats.eval_calls[name]++; } while (0) // Used by the _opcode extension which is built as a shared library PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -276,6 +279,8 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define CALL_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC_COND(name, cond) ((void)0) +#define EVAL_CALL_STAT_INC(name) ((void)0) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) #endif // !Py_STATS // Cache values are only valid in memory, so use native endianness. diff --git a/Include/pystats.h b/Include/pystats.h index 4375614b05e48..bea471bfc1de9 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -10,6 +10,20 @@ extern "C" { #define SPECIALIZATION_FAILURE_KINDS 32 +/* Stats for determining who is calling PyEval_EvalFrame */ +#define EVAL_CALL_TOTAL 0 +#define EVAL_CALL_VECTOR 1 +#define EVAL_CALL_GENERATOR 2 +#define EVAL_CALL_LEGACY 3 +#define EVAL_CALL_FUNCTION_VECTORCALL 4 +#define EVAL_CALL_BUILD_CLASS 5 +#define EVAL_CALL_SLOT 6 +#define EVAL_CALL_FUNCTION_EX 7 +#define EVAL_CALL_API 8 +#define EVAL_CALL_METHOD 9 + +#define EVAL_CALL_KINDS 10 + typedef struct _specialization_stats { uint64_t success; uint64_t failure; @@ -31,6 +45,7 @@ typedef struct _call_stats { uint64_t pyeval_calls; uint64_t frames_pushed; uint64_t frame_objects_created; + uint64_t eval_calls[EVAL_CALL_KINDS]; } CallStats; typedef struct _object_stats { diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c2b0e242e277f..bb7c5a7c3ec55 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -384,7 +384,7 @@ call_soon(PyObject *loop, PyObject *func, PyObject *arg, PyObject *ctx) nargs++; } stack[nargs] = (PyObject *)ctx; - + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, callable); handle = PyObject_Vectorcall(callable, stack, nargs, context_kwname); Py_DECREF(callable); } @@ -2950,6 +2950,7 @@ task_step_impl(TaskObj *task, PyObject *exc) PyObject *stack[2]; stack[0] = wrapper; stack[1] = (PyObject *)task->task_context; + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, add_cb); tmp = PyObject_Vectorcall(add_cb, stack, 1, context_kwname); Py_DECREF(add_cb); Py_DECREF(wrapper); diff --git a/Objects/call.c b/Objects/call.c index 678d16269f216..3b5259b3383f4 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -109,7 +109,9 @@ _Py_CheckSlotResult(PyObject *obj, const char *slot_name, int success) PyObject * PyObject_CallNoArgs(PyObject *func) { - return _PyObject_CallNoArgs(func); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); + PyThreadState *tstate = _PyThreadState_GET(); + return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } @@ -322,7 +324,7 @@ _PyObject_Call(PyThreadState *tstate, PyObject *callable, assert(!_PyErr_Occurred(tstate)); assert(PyTuple_Check(args)); assert(kwargs == NULL || PyDict_Check(kwargs)); - + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, callable); vectorcallfunc vector_func = _PyVectorcall_Function(callable); if (vector_func != NULL) { return _PyVectorcall_Call(tstate, vector_func, callable, args, kwargs); @@ -367,6 +369,7 @@ PyCFunction_Call(PyObject *callable, PyObject *args, PyObject *kwargs) PyObject * PyObject_CallOneArg(PyObject *func, PyObject *arg) { + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); assert(arg != NULL); PyObject *_args[2]; PyObject **args = _args + 1; // For PY_VECTORCALL_ARGUMENTS_OFFSET @@ -389,6 +392,7 @@ _PyFunction_Vectorcall(PyObject *func, PyObject* const* stack, assert(nargs >= 0); PyThreadState *tstate = _PyThreadState_GET(); assert(nargs == 0 || stack != NULL); + EVAL_CALL_STAT_INC(EVAL_CALL_FUNCTION_VECTORCALL); if (((PyCodeObject *)f->func_code)->co_flags & CO_OPTIMIZED) { return _PyEval_Vector(tstate, f, NULL, stack, nargs, kwnames); } @@ -520,7 +524,7 @@ _PyObject_CallFunctionVa(PyThreadState *tstate, PyObject *callable, if (stack == NULL) { return NULL; } - + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, callable); if (nargs == 1 && PyTuple_Check(stack[0])) { /* Special cases for backward compatibility: - PyObject_CallFunction(func, "O", tuple) calls func(*tuple) @@ -815,6 +819,11 @@ object_vacall(PyThreadState *tstate, PyObject *base, stack[i] = va_arg(vargs, PyObject *); } +#ifdef Py_STATS + if (PyFunction_Check(callable)) { + EVAL_CALL_STAT_INC(EVAL_CALL_API); + } +#endif /* Call the function */ result = _PyObject_VectorcallTstate(tstate, callable, stack, nargs, NULL); @@ -852,6 +861,7 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, args++; nargsf--; } + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable); PyObject *result = _PyObject_VectorcallTstate(tstate, callable, args, nargsf, kwnames); Py_DECREF(callable); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c3c541bf3c321..8e8a46ceca6b3 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1677,6 +1677,7 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) res = PyObject_CallOneArg(func, obj); } else { + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, func); PyObject *args[] = { obj, value }; res = PyObject_Vectorcall(func, args, 2, NULL); } diff --git a/Objects/genobject.c b/Objects/genobject.c index a88522abf414c..2b45e28cbf16d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -13,6 +13,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "structmember.h" // PyMemberDef #include "opcode.h" // SEND +#include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); @@ -218,6 +219,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, } gen->gi_frame_state = FRAME_EXECUTING; + EVAL_CALL_STAT_INC(EVAL_CALL_GENERATOR); result = _PyEval_EvalFrame(tstate, frame, exc); if (gen->gi_frame_state == FRAME_EXECUTING) { gen->gi_frame_state = FRAME_COMPLETED; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ff5196c904eef..37ac886f00446 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1653,6 +1653,7 @@ vectorcall_unbound(PyThreadState *tstate, int unbound, PyObject *func, args++; nargsf = nargsf - 1 + PY_VECTORCALL_ARGUMENTS_OFFSET; } + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_SLOT, func); return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 072bf75bf8d69..6284bbdf97468 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -198,6 +198,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto error; } PyThreadState *tstate = _PyThreadState_GET(); + EVAL_CALL_STAT_INC(EVAL_CALL_BUILD_CLASS); cell = _PyEval_Vector(tstate, (PyFunctionObject *)func, ns, NULL, 0, NULL); if (cell != NULL) { if (bases != orig_bases) { diff --git a/Python/ceval.c b/Python/ceval.c index 56b1017fd9298..6158524999a84 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1207,6 +1207,7 @@ PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) if (func == NULL) { return NULL; } + EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL); Py_DECREF(func); return res; @@ -6432,6 +6433,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, if (frame == NULL) { return NULL; } + EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0); assert( _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) || @@ -6507,6 +6509,7 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, if (func == NULL) { goto fail; } + EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); res = _PyEval_Vector(tstate, func, locals, allargs, argcount, kwnames); @@ -7299,7 +7302,6 @@ do_call_core(PyThreadState *tstate, ) { PyObject *result; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { C_TRACE(result, PyObject_Call(func, callargs, kwdict)); return result; @@ -7329,6 +7331,7 @@ do_call_core(PyThreadState *tstate, return result; } } + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); return PyObject_Call(func, callargs, kwdict); } diff --git a/Python/specialize.c b/Python/specialize.c index 7148b5dd02960..a2fb460388055 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -176,6 +176,9 @@ print_call_stats(FILE *out, CallStats *stats) fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); + } } static void @@ -904,7 +907,7 @@ typedef enum { MANAGED_DICT = 2, OFFSET_DICT = 3, NO_DICT = 4, - LAZY_DICT = 5, + LAZY_DICT = 5, } ObjectDictKind; // Please collect stats carefully before and after modifying. A subtle change diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 3d7479f261b4a..04d18790f0d79 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -108,13 +108,14 @@ def extract_opcode_stats(stats): opcode_stats[int(n)][rest.strip(".")] = value return opcode_stats -def parse_kinds(spec_src): +def parse_kinds(spec_src, prefix="SPEC_FAIL"): defines = collections.defaultdict(list) + start = "#define " + prefix + "_" for line in spec_src: line = line.strip() - if not line.startswith("#define SPEC_FAIL_"): + if not line.startswith(start): continue - line = line[len("#define SPEC_FAIL_"):] + line = line[len(start):] name, val = line.split() defines[int(val.strip())].append(name.strip()) return defines @@ -129,8 +130,6 @@ def kind_to_text(kind, defines, opname): opname = "ATTR" if opname.endswith("SUBSCR"): opname = "SUBSCR" - if opname.startswith("PRECALL"): - opname = "CALL" for name in defines[kind]: if name.startswith(opname): return pretty(name[len(opname)+1:]) @@ -254,6 +253,9 @@ def emit_specialization_overview(opcode_stats, total): )) def emit_call_stats(stats): + stats_path = os.path.join(os.path.dirname(__file__), "../../Include/pystats.h") + with open(stats_path) as stats_src: + defines = parse_kinds(stats_src, prefix="EVAL_CALL") with Section("Call stats", summary="Inlined calls and frame stats"): total = 0 for key, value in stats.items(): @@ -263,6 +265,11 @@ def emit_call_stats(stats): for key, value in stats.items(): if "Calls to" in key: rows.append((key, value, f"{100*value/total:0.1f}%")) + elif key.startswith("Calls "): + name, index = key[:-1].split("[") + index = int(index) + label = name + " (" + pretty(defines[index][0]) + ")" + rows.append((label, value, f"{100*value/total:0.1f}%")) for key, value in stats.items(): if key.startswith("Frame"): rows.append((key, value, f"{100*value/total:0.1f}%"))

1 0