Mailman 3 January 2025 - Python-checkins

[3.12] gh-111495: Add more tests on PyEval C APIs (#122789) (#128987) (#129023)
by vstinner Jan. 19, 2025

Jan. 19, 2025

https://github.com/python/cpython/commit/fceb8c31ddf8db2fddd92c8c6437094081… commit: fceb8c31ddf8db2fddd92c8c6437094081c91ac8 branch: 3.12 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-19T13:51:53Z summary: [3.12] gh-111495: Add more tests on PyEval C APIs (#122789) (#128987) (#129023) * Add Lib/test/test_capi/test_eval.py * Add Modules/_testlimitedcapi/eval.c (cherry picked from commit bf8b3746398ea756c97e3cf263d63ca3ce3a544e) * gh-111495: Fix refleaks in test_capi.test_eval tests (#122851) (cherry picked from commit b4a316087c32d83e375087fd35fc511bc430ee8b) (cherry picked from commit 430ccbc009aa7a2da92b85d7aeadd39e1666e875) files: A Lib/test/test_capi/test_eval.py A Modules/_testcapi/eval.c M Lib/test/test_capi/test_misc.py M Modules/Setup.stdlib.in M Modules/_testcapi/parts.h M Modules/_testcapimodule.c M PCbuild/_testcapi.vcxproj M PCbuild/_testcapi.vcxproj.filters diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py new file mode 100644 index 00000000000000..1fedf2f2b4b23b --- /dev/null +++ b/Lib/test/test_capi/test_eval.py @@ -0,0 +1,88 @@ +import sys +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class Tests(unittest.TestCase): + def test_eval_get_func_name(self): + eval_get_func_name = _testcapi.eval_get_func_name + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_name(function_example), + "function_example") + self.assertEqual(eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(eval_get_func_name(sum), "sum") # c function + self.assertEqual(eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + eval_get_func_desc = _testcapi.eval_get_func_desc + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_desc(function_example), + "()") + self.assertEqual(eval_get_func_desc(A.method_example), + "()") + self.assertEqual(eval_get_func_desc(A().method_example), + "()") + self.assertEqual(eval_get_func_desc(sum), "()") # c function + self.assertEqual(eval_get_func_desc(A), " object") + + def test_eval_getlocals(self): + # Test PyEval_GetLocals() + x = 1 + self.assertEqual(_testcapi.eval_getlocals(), + {'self': self, + 'x': 1}) + + y = 2 + self.assertEqual(_testcapi.eval_getlocals(), + {'self': self, + 'x': 1, + 'y': 2}) + + def test_eval_getglobals(self): + # Test PyEval_GetGlobals() + self.assertEqual(_testcapi.eval_getglobals(), + globals()) + + def test_eval_getbuiltins(self): + # Test PyEval_GetBuiltins() + self.assertEqual(_testcapi.eval_getbuiltins(), + globals()['__builtins__']) + + def test_eval_getframe(self): + # Test PyEval_GetFrame() + self.assertEqual(_testcapi.eval_getframe(), + sys._getframe()) + + def test_eval_get_recursion_limit(self): + # Test Py_GetRecursionLimit() + self.assertEqual(_testcapi.eval_get_recursion_limit(), + sys.getrecursionlimit()) + + def test_eval_set_recursion_limit(self): + # Test Py_SetRecursionLimit() + old_limit = sys.getrecursionlimit() + try: + limit = old_limit + 123 + _testcapi.eval_set_recursion_limit(limit) + self.assertEqual(sys.getrecursionlimit(), limit) + finally: + sys.setrecursionlimit(old_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 98c74a44e4cb27..f284e665d6fde0 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -879,36 +879,6 @@ def __init__(self): _testcapi.clear_managed_dict(c) self.assertEqual(c.__dict__, {}) - def test_eval_get_func_name(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_name(function_example), - "function_example") - self.assertEqual(_testcapi.eval_get_func_name(A.method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(A().method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function - self.assertEqual(_testcapi.eval_get_func_name(A), "type") - - def test_eval_get_func_desc(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_desc(function_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A.method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A().method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(sum), "()") # c function - self.assertEqual(_testcapi.eval_get_func_desc(A), " object") - def test_function_get_code(self): import types diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d60afcf55dba4a..5e587dd3a5a397 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -168,7 +168,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/import.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/import.c _testcapi/eval.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/eval.c b/Modules/_testcapi/eval.c new file mode 100644 index 00000000000000..581b5579178910 --- /dev/null +++ b/Modules/_testcapi/eval.c @@ -0,0 +1,74 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + +static PyObject * +eval_get_func_desc(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncDesc(func)); +} + +static PyObject * +eval_getlocals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetLocals()); +} + +static PyObject * +eval_getglobals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetGlobals()); +} + +static PyObject * +eval_getbuiltins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetBuiltins()); +} + +static PyObject * +eval_getframe(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetFrame()); +} + +static PyObject * +eval_get_recursion_limit(PyObject *module, PyObject *Py_UNUSED(args)) +{ + int limit = Py_GetRecursionLimit(); + return PyLong_FromLong(limit); +} + +static PyObject * +eval_set_recursion_limit(PyObject *module, PyObject *args) +{ + int limit; + if (!PyArg_ParseTuple(args, "i", &limit)) { + return NULL; + } + Py_SetRecursionLimit(limit); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, + {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, + {"eval_getlocals", eval_getlocals, METH_NOARGS}, + {"eval_getglobals", eval_getglobals, METH_NOARGS}, + {"eval_getbuiltins", eval_getbuiltins, METH_NOARGS}, + {"eval_getframe", eval_getframe, METH_NOARGS}, + {"eval_get_recursion_limit", eval_get_recursion_limit, METH_NOARGS}, + {"eval_set_recursion_limit", eval_set_recursion_limit, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Eval(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index bc348dc122c525..4ca46c800aa9eb 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -56,6 +56,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Import(PyObject *module); +int _PyTestCapi_Init_Eval(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 21d1228d79fe44..1fbeb58e3d404b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2797,18 +2797,6 @@ test_frame_getvarstring(PyObject *self, PyObject *args) } -static PyObject * -eval_get_func_name(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncName(func)); -} - -static PyObject * -eval_get_func_desc(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncDesc(func)); -} - static PyObject * gen_get_code(PyObject *self, PyObject *gen) { @@ -3460,8 +3448,6 @@ static PyMethodDef TestMethods[] = { {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, - {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, - {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"gen_get_code", gen_get_code, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, @@ -4174,6 +4160,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Import(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Eval(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index ec24c62903af90..06be7d291b17f3 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -128,6 +128,7 @@ <ClCompile Include="..\Modules\_testcapi\immortal.c" /> <ClCompile Include="..\Modules\_testcapi\gc.c" /> <ClCompile Include="..\Modules\_testcapi\run.c" /> + <ClCompile Include="..\Modules\_testcapi\eval.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 99eefb7b565393..309a016da2d587 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -111,6 +111,9 @@ <ClCompile Include="..\Modules\_testcapi\import.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\eval.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc">

1 0

[3.12] gh-128679: Fix tracemalloc.stop() race conditions (#128897) (#129022)
by vstinner Jan. 19, 2025

Jan. 19, 2025

https://github.com/python/cpython/commit/6df22cbf6049d09f45e214e74345a5159f… commit: 6df22cbf6049d09f45e214e74345a5159f7e202b branch: 3.12 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-19T13:24:14Z summary: [3.12] gh-128679: Fix tracemalloc.stop() race conditions (#128897) (#129022) [3.13] gh-128679: Fix tracemalloc.stop() race conditions (#128897) tracemalloc_alloc(), tracemalloc_realloc(), PyTraceMalloc_Track(), PyTraceMalloc_Untrack() and _PyTraceMalloc_TraceRef() now check tracemalloc_config.tracing after calling TABLES_LOCK(). _PyTraceMalloc_Stop() now protects more code with TABLES_LOCK(), especially setting tracemalloc_config.tracing to 1. Add a test using PyTraceMalloc_Track() to test tracemalloc.stop() race condition. Call _PyTraceMalloc_Init() at Python startup. (cherry picked from commit 6b47499510e47c0401d1f6cca2660fc12c496e39) files: A Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst M Include/tracemalloc.h M Lib/test/test_tracemalloc.py M Modules/_testcapimodule.c M Modules/_tracemalloc.c M Python/pylifecycle.c M Python/tracemalloc.c diff --git a/Include/tracemalloc.h b/Include/tracemalloc.h index 4db34b9509e7e2..2c88dbab7b8d35 100644 --- a/Include/tracemalloc.h +++ b/Include/tracemalloc.h @@ -50,7 +50,7 @@ PyAPI_FUNC(PyObject *) _PyTraceMalloc_GetTraces(void); PyAPI_FUNC(PyObject *) _PyTraceMalloc_GetObjectTraceback(PyObject *obj); /* Initialize tracemalloc */ -PyAPI_FUNC(int) _PyTraceMalloc_Init(void); +PyAPI_FUNC(PyStatus) _PyTraceMalloc_Init(void); /* Start tracemalloc */ PyAPI_FUNC(int) _PyTraceMalloc_Start(int max_nframe); diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 94bcee302fe730..97103a32e02ed0 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -8,6 +8,7 @@ interpreter_requires_environment) from test import support from test.support import os_helper +from test.support import threading_helper try: import _testcapi @@ -946,7 +947,6 @@ def check_env_var_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") - def test_env_var_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -1095,6 +1095,14 @@ def test_stop_untrack(self): with self.assertRaises(RuntimeError): self.untrack() + @unittest.skipIf(_testcapi is None, 'need _testcapi') + @threading_helper.requires_working_threading() + # gh-128679: Test crash on a debug build (especially on FreeBSD). + @unittest.skipIf(support.Py_DEBUG, 'need release build') + def test_tracemalloc_track_race(self): + # gh-128679: Test fix for tracemalloc.stop() race condition + _testcapi.tracemalloc_track_race() + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst new file mode 100644 index 00000000000000..837f90df07a705 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst @@ -0,0 +1,3 @@ +Fix :func:`tracemalloc.stop` race condition. Fix :mod:`tracemalloc` to +support calling :func:`tracemalloc.stop` in one thread, while another thread +is tracing memory allocations. Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 244bfab8d986f2..21d1228d79fe44 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3238,6 +3238,105 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); + +static void +tracemalloc_track_race_thread(void *data) +{ + PyTraceMalloc_Track(123, 10, 1); + + PyThread_type_lock lock = (PyThread_type_lock)data; + PyThread_release_lock(lock); +} + +// gh-128679: Test fix for tracemalloc.stop() race condition +static PyObject * +tracemalloc_track_race(PyObject *self, PyObject *args) +{ +#define NTHREAD 50 + PyObject *tracemalloc = NULL; + PyObject *stop = NULL; + PyThread_type_lock locks[NTHREAD]; + memset(locks, 0, sizeof(locks)); + + // Call tracemalloc.start() + tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc == NULL) { + goto error; + } + PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); + if (start == NULL) { + goto error; + } + PyObject *res = PyObject_CallNoArgs(start); + Py_DECREF(start); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + stop = PyObject_GetAttrString(tracemalloc, "stop"); + Py_CLEAR(tracemalloc); + if (stop == NULL) { + goto error; + } + + // Start threads + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + PyErr_NoMemory(); + goto error; + } + locks[i] = lock; + PyThread_acquire_lock(lock, 1); + + unsigned long thread; + thread = PyThread_start_new_thread(tracemalloc_track_race_thread, + (void*)lock); + if (thread == (unsigned long)-1) { + PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); + goto error; + } + } + + // Call tracemalloc.stop() while threads are running + res = PyObject_CallNoArgs(stop); + Py_CLEAR(stop); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + // Wait until threads complete with the GIL released + Py_BEGIN_ALLOW_THREADS + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_acquire_lock(lock, 1); + PyThread_release_lock(lock); + } + Py_END_ALLOW_THREADS + + // Free threads locks + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_free_lock(lock); + } + Py_RETURN_NONE; + +error: + Py_CLEAR(tracemalloc); + Py_CLEAR(stop); + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + if (lock) { + PyThread_free_lock(lock); + } + } + return NULL; +#undef NTHREAD +} + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3378,6 +3477,7 @@ static PyMethodDef TestMethods[] = { {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"test_atexit", test_atexit, METH_NOARGS}, + {"tracemalloc_track_race", tracemalloc_track_race, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index f3f4af9aba08c1..3a4092c9d0859b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -219,10 +219,5 @@ PyInit__tracemalloc(void) if (m == NULL) return NULL; - if (_PyTraceMalloc_Init() < 0) { - Py_DECREF(m); - return NULL; - } - return m; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e9c1a0d72d6d4d..f301026fa5befd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -654,6 +654,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime, // didn't depend on interp->feature_flags being set already. _PyObject_InitState(interp); + status = _PyTraceMalloc_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + PyThreadState *tstate = _PyThreadState_New(interp); if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index e13064bd145e8e..9e653770c3eab5 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() #include "pycore_object.h" // _PyType_PreHeaderSize #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() @@ -536,12 +537,16 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (ADD_TRACE(ptr, nelem * elsize) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); - alloc->free(alloc->ctx, ptr); - return NULL; + + if (tracemalloc_config.tracing) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { + /* Failed to allocate a trace for the new memory block */ + alloc->free(alloc->ctx, ptr); + ptr = NULL; + } } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); return ptr; } @@ -557,11 +562,15 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 == NULL) return NULL; + TABLES_LOCK(); + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto done; + } + if (ptr != NULL) { /* an existing memory block has been resized */ - TABLES_LOCK(); - /* tracemalloc_add_trace() updates the trace if there is already a trace at address ptr2 */ if (ptr2 != ptr) { @@ -580,20 +589,19 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) allocating memory. */ Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } - TABLES_UNLOCK(); } else { /* new allocation */ - TABLES_LOCK(); if (ADD_TRACE(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); - return NULL; + ptr2 = NULL; } - TABLES_UNLOCK(); } + +done: + TABLES_UNLOCK(); return ptr2; } @@ -612,7 +620,12 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - REMOVE_TRACE(ptr); + + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); } @@ -671,7 +684,9 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - REMOVE_TRACE(ptr); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } TABLES_UNLOCK(); } return ptr2; @@ -746,7 +761,9 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - REMOVE_TRACE(ptr); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } TABLES_UNLOCK(); } return ptr2; @@ -777,46 +794,36 @@ tracemalloc_clear_filename(void *value) /* reentrant flag must be set to call this function and GIL must be held */ static void -tracemalloc_clear_traces(void) +tracemalloc_clear_traces_unlocked(void) { + set_reentrant(1); + /* The GIL protects variables against concurrent access */ assert(PyGILState_Check()); - TABLES_LOCK(); _Py_hashtable_clear(tracemalloc_traces); _Py_hashtable_clear(tracemalloc_domains); tracemalloc_traced_memory = 0; tracemalloc_peak_traced_memory = 0; - TABLES_UNLOCK(); _Py_hashtable_clear(tracemalloc_tracebacks); _Py_hashtable_clear(tracemalloc_filenames); + + set_reentrant(0); } -int +PyStatus _PyTraceMalloc_Init(void) { - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { - PyErr_SetString(PyExc_RuntimeError, - "the tracemalloc module has been unloaded"); - return -1; - } - - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) - return 0; + assert(tracemalloc_config.initialized == TRACEMALLOC_NOT_INITIALIZED); PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { -#ifdef MS_WINDOWS - PyErr_SetFromWindowsErr(0); -#else - PyErr_SetFromErrno(PyExc_OSError); -#endif - return -1; + return _PyStatus_NO_MEMORY(); } #endif @@ -824,8 +831,7 @@ _PyTraceMalloc_Init(void) if (tables_lock == NULL) { tables_lock = PyThread_allocate_lock(); if (tables_lock == NULL) { - PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); - return -1; + return _PyStatus_NO_MEMORY(); } } #endif @@ -842,9 +848,9 @@ _PyTraceMalloc_Init(void) tracemalloc_domains = tracemalloc_create_domains_table(); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL || tracemalloc_domains == NULL) { - PyErr_NoMemory(); - return -1; + || tracemalloc_traces == NULL || tracemalloc_domains == NULL) + { + return _PyStatus_NO_MEMORY(); } tracemalloc_empty_traceback.nframe = 1; @@ -855,7 +861,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; - return 0; + return _PyStatus_OK(); } @@ -900,10 +906,6 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - if (_PyTraceMalloc_Init() < 0) { - return -1; - } - if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; @@ -954,8 +956,13 @@ _PyTraceMalloc_Start(int max_nframe) void _PyTraceMalloc_Stop(void) { - if (!tracemalloc_config.tracing) - return; + // Lock to synchronize with tracemalloc_free() which checks + // 'tracing' while holding the lock. + TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + goto done; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -967,11 +974,14 @@ _PyTraceMalloc_Stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - tracemalloc_clear_traces(); + tracemalloc_clear_traces_unlocked(); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; + +done: + TABLES_UNLOCK(); } @@ -1221,23 +1231,17 @@ tracemalloc_pyobject_decref(void *value) static traceback_t* -tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) +tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) { - - if (!tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) { return NULL; + } - trace_t *trace; - TABLES_LOCK(); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); + trace_t *trace = NULL; if (traces) { trace = _Py_hashtable_get(traces, TO_PTR(ptr)); } - else { - trace = NULL; - } - TABLES_UNLOCK(); - if (!trace) { return NULL; } @@ -1266,13 +1270,20 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + if (tracemalloc_config.tracing) { + traceback = tracemalloc_get_traceback_unlocked(DEFAULT_DOMAIN, + (uintptr_t)ptr); + } + else { + traceback = NULL; PUTS(fd, "Enable tracemalloc to get the memory block " "allocation traceback\n\n"); - return; } - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr); + TABLES_UNLOCK(); + if (traceback == NULL) return; @@ -1301,20 +1312,19 @@ int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { - int res; - PyGILState_STATE gil_state; + PyGILState_STATE gil_state = PyGILState_Ensure(); + TABLES_LOCK(); - if (!tracemalloc_config.tracing) { - /* tracemalloc is not tracing: do nothing */ - return -2; + int res; + if (tracemalloc_config.tracing) { + res = tracemalloc_add_trace(domain, ptr, size); + } + else { + // gh-128679: tracemalloc.stop() was called by another thread + res = -2; } - gil_state = PyGILState_Ensure(); - - TABLES_LOCK(); - res = tracemalloc_add_trace(domain, ptr, size); TABLES_UNLOCK(); - PyGILState_Release(gil_state); return res; } @@ -1323,16 +1333,20 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + int result; + if (tracemalloc_config.tracing) { + tracemalloc_remove_trace(domain, ptr); + result = 0; + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - TABLES_LOCK(); - tracemalloc_remove_trace(domain, ptr); TABLES_UNLOCK(); - - return 0; + return result; } @@ -1366,6 +1380,12 @@ _PyTraceMalloc_NewReference(PyObject *op) int res = -1; TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto done; + } + trace_t *trace = _Py_hashtable_get(tracemalloc_traces, TO_PTR(ptr)); if (trace != NULL) { /* update the traceback of the memory block */ @@ -1376,6 +1396,8 @@ _PyTraceMalloc_NewReference(PyObject *op) } } /* else: cannot track the object, its memory block size is unknown */ + +done: TABLES_UNLOCK(); return res; @@ -1387,7 +1409,9 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) { traceback_t *traceback; - traceback = tracemalloc_get_traceback(domain, ptr); + TABLES_LOCK(); + traceback = tracemalloc_get_traceback_unlocked(domain, ptr); + TABLES_UNLOCK(); if (traceback == NULL) Py_RETURN_NONE; @@ -1397,19 +1421,20 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) int _PyTraceMalloc_IsTracing(void) { - return tracemalloc_config.tracing; + TABLES_LOCK(); + int tracing = tracemalloc_config.tracing; + TABLES_UNLOCK(); + return tracing; } void _PyTraceMalloc_ClearTraces(void) { - - if (!tracemalloc_config.tracing) { - return; + TABLES_LOCK(); + if (tracemalloc_config.tracing) { + tracemalloc_clear_traces_unlocked(); } - set_reentrant(1); - tracemalloc_clear_traces(); - set_reentrant(0); + TABLES_UNLOCK(); } PyObject * @@ -1496,19 +1521,10 @@ PyObject * _PyTraceMalloc_GetObjectTraceback(PyObject *obj) /*[clinic end generated code: output=41ee0553a658b0aa input=29495f1b21c53212]*/ { - PyTypeObject *type; - traceback_t *traceback; - - type = Py_TYPE(obj); + PyTypeObject *type = Py_TYPE(obj); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)obj - presize); - - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); - if (traceback == NULL) { - Py_RETURN_NONE; - } - - return traceback_to_pyobject(traceback, NULL); + return _PyTraceMalloc_GetTraceback(DEFAULT_DOMAIN, ptr); } int _PyTraceMalloc_GetTracebackLimit(void) { @@ -1520,14 +1536,19 @@ _PyTraceMalloc_GetMemory(void) { size_t size; - size = _Py_hashtable_size(tracemalloc_tracebacks); - size += _Py_hashtable_size(tracemalloc_filenames); - TABLES_LOCK(); - size += _Py_hashtable_size(tracemalloc_traces); - _Py_hashtable_foreach(tracemalloc_domains, - tracemalloc_get_tracemalloc_memory_cb, &size); + if (tracemalloc_config.tracing) { + size = _Py_hashtable_size(tracemalloc_tracebacks); + size += _Py_hashtable_size(tracemalloc_filenames); + size += _Py_hashtable_size(tracemalloc_traces); + _Py_hashtable_foreach(tracemalloc_domains, + tracemalloc_get_tracemalloc_memory_cb, &size); + } + else { + size = 0; + } TABLES_UNLOCK(); + return size; } @@ -1537,12 +1558,15 @@ _PyTraceMalloc_GetTracedMemory(void) { Py_ssize_t size, peak_size; - if (!tracemalloc_config.tracing) - return Py_BuildValue("ii", 0, 0); - TABLES_LOCK(); - size = tracemalloc_traced_memory; - peak_size = tracemalloc_peak_traced_memory; + if (tracemalloc_config.tracing) { + size = tracemalloc_traced_memory; + peak_size = tracemalloc_peak_traced_memory; + } + else { + size = 0; + peak_size = 0; + } TABLES_UNLOCK(); return Py_BuildValue("nn", size, peak_size); @@ -1551,10 +1575,9 @@ _PyTraceMalloc_GetTracedMemory(void) void _PyTraceMalloc_ResetPeak(void) { - if (!tracemalloc_config.tracing) { - return; - } TABLES_LOCK(); - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.tracing) { + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + } TABLES_UNLOCK(); }

1 0

[3.12] gh-128911: Add tests on the PyImport C API (GH-128915) (GH-128960) (#128989)
by vstinner Jan. 19, 2025

Jan. 19, 2025

https://github.com/python/cpython/commit/83de72e5ec9522a1b01d8259f574efa0d6… commit: 83de72e5ec9522a1b01d8259f574efa0d6128d6a branch: 3.12 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-19T14:21:55+01:00 summary: [3.12] gh-128911: Add tests on the PyImport C API (GH-128915) (GH-128960) (#128989) * Add Modules/_testlimitedcapi/import.c * Add Lib/test/test_capi/test_import.py * Remove _testcapi.check_pyimport_addmodule(): tests already covered by newly added tests. (cherry picked from commit 34ded1a1a10204635cad27830fcbee2f8547e8ed) Co-authored-by: Victor Stinner <vstinner(a)python.org> Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> (cherry picked from commit d95ba9fa1110534b7247fa2ff12b90e930c93256) files: A Lib/test/test_capi/test_import.py A Modules/_testcapi/import.c M Modules/Setup.stdlib.in M Modules/_testcapi/parts.h M Modules/_testcapimodule.c M PCbuild/_testcapi.vcxproj M PCbuild/_testcapi.vcxproj.filters diff --git a/Lib/test/test_capi/test_import.py b/Lib/test/test_capi/test_import.py new file mode 100644 index 00000000000000..bcecd6e8cc69ff --- /dev/null +++ b/Lib/test/test_capi/test_import.py @@ -0,0 +1,313 @@ +import importlib.util +import os.path +import sys +import types +import unittest +from test.support import os_helper +from test.support import import_helper +from test.support.warnings_helper import check_warnings + +_testcapi = import_helper.import_module('_testcapi') +NULL = None + + +class ImportTests(unittest.TestCase): + def test_getmagicnumber(self): + # Test PyImport_GetMagicNumber() + magic = _testcapi.PyImport_GetMagicNumber() + self.assertEqual(magic, + int.from_bytes(importlib.util.MAGIC_NUMBER, 'little')) + + def test_getmagictag(self): + # Test PyImport_GetMagicTag() + tag = _testcapi.PyImport_GetMagicTag() + self.assertEqual(tag, sys.implementation.cache_tag) + + def test_getmoduledict(self): + # Test PyImport_GetModuleDict() + modules = _testcapi.PyImport_GetModuleDict() + self.assertIs(modules, sys.modules) + + def check_import_loaded_module(self, import_module): + for name in ('os', 'sys', 'test', 'unittest'): + with self.subTest(name=name): + self.assertIn(name, sys.modules) + old_module = sys.modules[name] + module = import_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertIs(module, old_module) + + def check_import_fresh_module(self, import_module): + old_modules = dict(sys.modules) + try: + for name in ('colorsys', 'math'): + with self.subTest(name=name): + sys.modules.pop(name, None) + module = import_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertIs(module, sys.modules[name]) + self.assertEqual(module.__name__, name) + finally: + sys.modules.clear() + sys.modules.update(old_modules) + + def test_getmodule(self): + # Test PyImport_GetModule() + getmodule = _testcapi.PyImport_GetModule + self.check_import_loaded_module(getmodule) + + nonexistent = 'nonexistent' + self.assertNotIn(nonexistent, sys.modules) + self.assertIs(getmodule(nonexistent), KeyError) + self.assertIs(getmodule(''), KeyError) + self.assertIs(getmodule(object()), KeyError) + + self.assertRaises(TypeError, getmodule, []) # unhashable + # CRASHES getmodule(NULL) + + def check_addmodule(self, add_module, accept_nonstr=False): + # create a new module + names = ['nonexistent'] + if accept_nonstr: + names.append(b'\xff') # non-UTF-8 + for name in names: + with self.subTest(name=name): + self.assertNotIn(name, sys.modules) + try: + module = add_module(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + self.assertIs(module, sys.modules[name]) + finally: + sys.modules.pop(name, None) + + # get an existing module + self.check_import_loaded_module(add_module) + + def test_addmoduleobject(self): + # Test PyImport_AddModuleObject() + addmoduleobject = _testcapi.PyImport_AddModuleObject + self.check_addmodule(addmoduleobject, accept_nonstr=True) + + self.assertRaises(TypeError, addmoduleobject, []) # unhashable + # CRASHES addmoduleobject(NULL) + + def test_addmodule(self): + # Test PyImport_AddModule() + addmodule = _testcapi.PyImport_AddModule + self.check_addmodule(addmodule) + + self.assertRaises(UnicodeDecodeError, addmodule, b'\xff') + # CRASHES addmodule(NULL) + + def check_import_func(self, import_module): + self.check_import_loaded_module(import_module) + self.check_import_fresh_module(import_module) + self.assertRaises(ModuleNotFoundError, import_module, 'nonexistent') + self.assertRaises(ValueError, import_module, '') + + def test_import(self): + # Test PyImport_Import() + import_ = _testcapi.PyImport_Import + self.check_import_func(import_) + + self.assertRaises(TypeError, import_, b'os') + self.assertRaises(SystemError, import_, NULL) + + def test_importmodule(self): + # Test PyImport_ImportModule() + importmodule = _testcapi.PyImport_ImportModule + self.check_import_func(importmodule) + + self.assertRaises(UnicodeDecodeError, importmodule, b'\xff') + # CRASHES importmodule(NULL) + + def test_importmodulenoblock(self): + # Test deprecated PyImport_ImportModuleNoBlock() + importmodulenoblock = _testcapi.PyImport_ImportModuleNoBlock + self.check_import_func(importmodulenoblock) + + self.assertRaises(UnicodeDecodeError, importmodulenoblock, b'\xff') + # CRASHES importmodulenoblock(NULL) + + def check_frozen_import(self, import_frozen_module): + # Importing a frozen module executes its code, so start by unloading + # the module to execute the code in a new (temporary) module. + old_zipimport = sys.modules.pop('zipimport') + try: + self.assertEqual(import_frozen_module('zipimport'), 1) + + # import zipimport again + self.assertEqual(import_frozen_module('zipimport'), 1) + finally: + sys.modules['zipimport'] = old_zipimport + + # not a frozen module + self.assertEqual(import_frozen_module('sys'), 0) + self.assertEqual(import_frozen_module('nonexistent'), 0) + self.assertEqual(import_frozen_module(''), 0) + + def test_importfrozenmodule(self): + # Test PyImport_ImportFrozenModule() + importfrozenmodule = _testcapi.PyImport_ImportFrozenModule + self.check_frozen_import(importfrozenmodule) + + self.assertRaises(UnicodeDecodeError, importfrozenmodule, b'\xff') + # CRASHES importfrozenmodule(NULL) + + def test_importfrozenmoduleobject(self): + # Test PyImport_ImportFrozenModuleObject() + importfrozenmoduleobject = _testcapi.PyImport_ImportFrozenModuleObject + self.check_frozen_import(importfrozenmoduleobject) + self.assertEqual(importfrozenmoduleobject(b'zipimport'), 0) + self.assertEqual(importfrozenmoduleobject(NULL), 0) + + def test_importmoduleex(self): + # Test PyImport_ImportModuleEx() + importmoduleex = _testcapi.PyImport_ImportModuleEx + self.check_import_func(lambda name: importmoduleex(name, NULL, NULL, NULL)) + + self.assertRaises(ModuleNotFoundError, importmoduleex, 'nonexistent', NULL, NULL, NULL) + self.assertRaises(ValueError, importmoduleex, '', NULL, NULL, NULL) + self.assertRaises(UnicodeDecodeError, importmoduleex, b'\xff', NULL, NULL, NULL) + # CRASHES importmoduleex(NULL, NULL, NULL, NULL) + + def check_importmodulelevel(self, importmodulelevel): + self.check_import_func(lambda name: importmodulelevel(name, NULL, NULL, NULL, 0)) + + self.assertRaises(ModuleNotFoundError, importmodulelevel, 'nonexistent', NULL, NULL, NULL, 0) + self.assertRaises(ValueError, importmodulelevel, '', NULL, NULL, NULL, 0) + + if __package__: + self.assertIs(importmodulelevel('test_import', globals(), NULL, NULL, 1), + sys.modules['test.test_capi.test_import']) + self.assertIs(importmodulelevel('test_capi', globals(), NULL, NULL, 2), + sys.modules['test.test_capi']) + self.assertRaises(ValueError, importmodulelevel, 'os', NULL, NULL, NULL, -1) + with self.assertWarns(ImportWarning): + self.assertRaises(KeyError, importmodulelevel, 'test_import', {}, NULL, NULL, 1) + self.assertRaises(TypeError, importmodulelevel, 'test_import', [], NULL, NULL, 1) + + def test_importmodulelevel(self): + # Test PyImport_ImportModuleLevel() + importmodulelevel = _testcapi.PyImport_ImportModuleLevel + self.check_importmodulelevel(importmodulelevel) + + self.assertRaises(UnicodeDecodeError, importmodulelevel, b'\xff', NULL, NULL, NULL, 0) + # CRASHES importmodulelevel(NULL, NULL, NULL, NULL, 0) + + def test_importmodulelevelobject(self): + # Test PyImport_ImportModuleLevelObject() + importmodulelevel = _testcapi.PyImport_ImportModuleLevelObject + self.check_importmodulelevel(importmodulelevel) + + self.assertRaises(TypeError, importmodulelevel, b'os', NULL, NULL, NULL, 0) + self.assertRaises(ValueError, importmodulelevel, NULL, NULL, NULL, NULL, 0) + + def check_executecodemodule(self, execute_code, *args): + name = 'test_import_executecode' + try: + # Create a temporary module where the code will be executed + self.assertNotIn(name, sys.modules) + module = _testcapi.PyImport_AddModule(name) + self.assertFalse(hasattr(module, 'attr')) + + # Execute the code + code = compile('attr = 1', '<test>', 'exec') + module2 = execute_code(name, code, *args) + self.assertIs(module2, module) + + # Check the function side effects + self.assertEqual(module.attr, 1) + finally: + sys.modules.pop(name, None) + return module.__spec__.origin + + def test_executecodemodule(self): + # Test PyImport_ExecCodeModule() + execcodemodule = _testcapi.PyImport_ExecCodeModule + self.check_executecodemodule(execcodemodule) + + code = compile('attr = 1', '<test>', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemodule, b'\xff', code) + # CRASHES execcodemodule(NULL, code) + # CRASHES execcodemodule(name, NULL) + + def test_executecodemoduleex(self): + # Test PyImport_ExecCodeModuleEx() + execcodemoduleex = _testcapi.PyImport_ExecCodeModuleEx + + # Test NULL path (it should not crash) + self.check_executecodemodule(execcodemoduleex, NULL) + + # Test non-NULL path + pathname = b'pathname' + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + code = compile('attr = 1', '<test>', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemoduleex, b'\xff', code, NULL) + # CRASHES execcodemoduleex(NULL, code, NULL) + # CRASHES execcodemoduleex(name, NULL, NULL) + + def check_executecode_pathnames(self, execute_code_func, object=False): + # Test non-NULL pathname and NULL cpathname + + # Test NULL paths (it should not crash) + self.check_executecodemodule(execute_code_func, NULL, NULL) + + pathname = 'pathname' + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + origin = self.check_executecodemodule(execute_code_func, NULL, pathname) + if not object: + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + if object: + pathname = os.fsdecode(pathname) + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + self.check_executecodemodule(execute_code_func, NULL, pathname) + + # Test NULL pathname and non-NULL cpathname + pyc_filename = importlib.util.cache_from_source(__file__) + py_filename = importlib.util.source_from_cache(pyc_filename) + origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename) + if not object: + self.assertEqual(origin, py_filename) + + def test_executecodemodulewithpathnames(self): + # Test PyImport_ExecCodeModuleWithPathnames() + execute_code_func = _testcapi.PyImport_ExecCodeModuleWithPathnames + self.check_executecode_pathnames(execute_code_func) + + code = compile('attr = 1', '<test>', 'exec') + self.assertRaises(UnicodeDecodeError, execute_code_func, b'\xff', code, NULL, NULL) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + def test_executecodemoduleobject(self): + # Test PyImport_ExecCodeModuleObject() + execute_code_func = _testcapi.PyImport_ExecCodeModuleObject + self.check_executecode_pathnames(execute_code_func, object=True) + + code = compile('attr = 1', '<test>', 'exec') + self.assertRaises(TypeError, execute_code_func, [], code, NULL, NULL) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + # TODO: test PyImport_GetImporter() + # TODO: test PyImport_ReloadModule() + # TODO: test PyImport_ExtendInittab() + # PyImport_AppendInittab() is tested by test_embed + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b68b8e4e347bbd..d60afcf55dba4a 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -168,7 +168,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/import.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/import.c b/Modules/_testcapi/import.c new file mode 100644 index 00000000000000..c6b5caa65f31e1 --- /dev/null +++ b/Modules/_testcapi/import.c @@ -0,0 +1,287 @@ +#define PY_SSIZE_T_CLEAN + +#include "parts.h" +#include "util.h" + + +/* Test PyImport_GetMagicNumber() */ +static PyObject * +pyimport_getmagicnumber(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + long magic = PyImport_GetMagicNumber(); + return PyLong_FromLong(magic); +} + + +/* Test PyImport_GetMagicTag() */ +static PyObject * +pyimport_getmagictag(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + const char *tag = PyImport_GetMagicTag(); + return PyUnicode_FromString(tag); +} + + +/* Test PyImport_GetModuleDict() */ +static PyObject * +pyimport_getmoduledict(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyImport_GetModuleDict()); +} + + +/* Test PyImport_GetModule() */ +static PyObject * +pyimport_getmodule(PyObject *Py_UNUSED(module), PyObject *name) +{ + assert(!PyErr_Occurred()); + NULLABLE(name); + PyObject *module = PyImport_GetModule(name); + if (module == NULL && !PyErr_Occurred()) { + return Py_NewRef(PyExc_KeyError); + } + return module; +} + + +/* Test PyImport_AddModuleObject() */ +static PyObject * +pyimport_addmoduleobject(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return Py_XNewRef(PyImport_AddModuleObject(name)); +} + + +/* Test PyImport_AddModule() */ +static PyObject * +pyimport_addmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + return Py_XNewRef(PyImport_AddModule(name)); +} + + +/* Test PyImport_Import() */ +static PyObject * +pyimport_import(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return PyImport_Import(name); +} + + +/* Test PyImport_ImportModule() */ +static PyObject * +pyimport_importmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + return PyImport_ImportModule(name); +} + + +/* Test PyImport_ImportModuleNoBlock() */ +static PyObject * +pyimport_importmodulenoblock(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS + return PyImport_ImportModuleNoBlock(name); + _Py_COMP_DIAG_POP +} + + +/* Test PyImport_ImportModuleEx() */ +static PyObject * +pyimport_importmoduleex(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *globals, *locals, *fromlist; + if (!PyArg_ParseTuple(args, "z#OOO", + &name, &size, &globals, &locals, &fromlist)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleEx(name, globals, locals, fromlist); +} + + +/* Test PyImport_ImportModuleLevel() */ +static PyObject * +pyimport_importmodulelevel(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *globals, *locals, *fromlist; + int level; + if (!PyArg_ParseTuple(args, "z#OOOi", + &name, &size, &globals, &locals, &fromlist, &level)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleLevel(name, globals, locals, fromlist, level); +} + + +/* Test PyImport_ImportModuleLevelObject() */ +static PyObject * +pyimport_importmodulelevelobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *name, *globals, *locals, *fromlist; + int level; + if (!PyArg_ParseTuple(args, "OOOOi", + &name, &globals, &locals, &fromlist, &level)) { + return NULL; + } + NULLABLE(name); + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); +} + + +/* Test PyImport_ImportFrozenModule() */ +static PyObject * +pyimport_importfrozenmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + RETURN_INT(PyImport_ImportFrozenModule(name)); +} + + +/* Test PyImport_ImportFrozenModuleObject() */ +static PyObject * +pyimport_importfrozenmoduleobject(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + RETURN_INT(PyImport_ImportFrozenModuleObject(name)); +} + + +/* Test PyImport_ExecCodeModule() */ +static PyObject * +pyimport_executecodemodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &code)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModule(name, code); +} + + +/* Test PyImport_ExecCodeModuleEx() */ +static PyObject * +pyimport_executecodemoduleex(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + const char *pathname; + if (!PyArg_ParseTuple(args, "z#Oz#", &name, &size, &code, &pathname, &size)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModuleEx(name, code, pathname); +} + + +/* Test PyImport_ExecCodeModuleWithPathnames() */ +static PyObject * +pyimport_executecodemodulewithpathnames(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + const char *pathname; + const char *cpathname; + if (!PyArg_ParseTuple(args, "z#Oz#z#", &name, &size, &code, &pathname, &size, &cpathname, &size)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModuleWithPathnames(name, code, + pathname, cpathname); +} + + +/* Test PyImport_ExecCodeModuleObject() */ +static PyObject * +pyimport_executecodemoduleobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *name, *code, *pathname, *cpathname; + if (!PyArg_ParseTuple(args, "OOOO", &name, &code, &pathname, &cpathname)) { + return NULL; + } + NULLABLE(name); + NULLABLE(code); + NULLABLE(pathname); + NULLABLE(cpathname); + + return PyImport_ExecCodeModuleObject(name, code, pathname, cpathname); +} + + +static PyMethodDef test_methods[] = { + {"PyImport_GetMagicNumber", pyimport_getmagicnumber, METH_NOARGS}, + {"PyImport_GetMagicTag", pyimport_getmagictag, METH_NOARGS}, + {"PyImport_GetModuleDict", pyimport_getmoduledict, METH_NOARGS}, + {"PyImport_GetModule", pyimport_getmodule, METH_O}, + {"PyImport_AddModuleObject", pyimport_addmoduleobject, METH_O}, + {"PyImport_AddModule", pyimport_addmodule, METH_VARARGS}, + {"PyImport_Import", pyimport_import, METH_O}, + {"PyImport_ImportModule", pyimport_importmodule, METH_VARARGS}, + {"PyImport_ImportModuleNoBlock", pyimport_importmodulenoblock, METH_VARARGS}, + {"PyImport_ImportModuleEx", pyimport_importmoduleex, METH_VARARGS}, + {"PyImport_ImportModuleLevel", pyimport_importmodulelevel, METH_VARARGS}, + {"PyImport_ImportModuleLevelObject", pyimport_importmodulelevelobject, METH_VARARGS}, + {"PyImport_ImportFrozenModule", pyimport_importfrozenmodule, METH_VARARGS}, + {"PyImport_ImportFrozenModuleObject", pyimport_importfrozenmoduleobject, METH_O}, + {"PyImport_ExecCodeModule", pyimport_executecodemodule, METH_VARARGS}, + {"PyImport_ExecCodeModuleEx", pyimport_executecodemoduleex, METH_VARARGS}, + {"PyImport_ExecCodeModuleWithPathnames", pyimport_executecodemodulewithpathnames, METH_VARARGS}, + {"PyImport_ExecCodeModuleObject", pyimport_executecodemoduleobject, METH_VARARGS}, + {NULL}, +}; + + +int +_PyTestCapi_Init_Import(PyObject *module) +{ + return PyModule_AddFunctions(module, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 496dd38fbad0bc..bc348dc122c525 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -55,6 +55,7 @@ int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); +int _PyTestCapi_Init_Import(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index aece635554daa0..244bfab8d986f2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4071,6 +4071,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_GC(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Import(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 257562c75d103b..ec24c62903af90 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -105,6 +105,7 @@ <ClCompile Include="..\Modules\_testcapi\unicode.c" /> <ClCompile Include="..\Modules\_testcapi\dict.c" /> <ClCompile Include="..\Modules\_testcapi\set.c" /> + <ClCompile Include="..\Modules\_testcapi\import.c" /> <ClCompile Include="..\Modules\_testcapi\list.c" /> <ClCompile Include="..\Modules\_testcapi\tuple.c" /> <ClCompile Include="..\Modules\_testcapi\pytime.c" /> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 4d1e4330d43989..99eefb7b565393 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -108,6 +108,9 @@ <ClCompile Include="..\Modules\_testcapi\run.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\import.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc">

1 0

gh-100239: Handle NaN and zero division in guards for `BINARY_OP_EXTEND` (#128963)
by Eclips4 Jan. 19, 2025

Jan. 19, 2025

https://github.com/python/cpython/commit/6c52ada5512ccd1a0891c00cc84c7d3170… commit: 6c52ada5512ccd1a0891c00cc84c7d3170d3328c branch: main author: Kirill Podoprigora <kirill.bast9(a)mail.ru> committer: Eclips4 <kirill.bast9(a)mail.ru> date: 2025-01-19T11:02:49Z summary: gh-100239: Handle NaN and zero division in guards for `BINARY_OP_EXTEND` (#128963) Co-authored-by: Tomas R. <tomas.roun8(a)gmail.com> Co-authored-by: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> files: M Lib/test/test_opcache.py M Python/specialize.c diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 72b845fcc8fdbf..4d7304b1c9abb6 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1362,6 +1362,53 @@ def binary_op_add_extend(): self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + def binary_op_zero_division(): + def compactlong_lhs(arg): + 42 / arg + def float_lhs(arg): + 42.0 / arg + + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0) + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(float_lhs, "BINARY_OP_EXTEND") + + binary_op_zero_division() + + def binary_op_nan(): + def compactlong_lhs(arg): + return ( + 42 + arg, + 42 - arg, + 42 * arg, + 42 / arg, + ) + def compactlong_rhs(arg): + return ( + arg + 42, + arg - 42, + arg * 2, + arg / 42, + ) + nan = float('nan') + self.assertEqual(compactlong_lhs(1.0), (43.0, 41.0, 42.0, 42.0)) + for _ in range(100): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_lhs(nan)))) + self.assertEqual(compactlong_rhs(42.0), (84.0, 0.0, 84.0, 1.0)) + for _ in range(100): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_rhs(nan)))) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(compactlong_rhs, "BINARY_OP_EXTEND") + + binary_op_nan() @cpython_only @requires_specialization_ft diff --git a/Python/specialize.c b/Python/specialize.c index 09bfcd34b5a543..fa022346bdea6a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2416,16 +2416,25 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) /* float-long */ -static int +static inline int float_compactlong_guard(PyObject *lhs, PyObject *rhs) { return ( PyFloat_CheckExact(lhs) && + !isnan(PyFloat_AsDouble(lhs)) && PyLong_CheckExact(rhs) && _PyLong_IsCompact((PyLongObject *)rhs) ); } +static inline int +nonzero_float_compactlong_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + float_compactlong_guard(lhs, rhs) && !PyLong_IsZero(rhs) + ); +} + #define FLOAT_LONG_ACTION(NAME, OP) \ static PyObject * \ (NAME)(PyObject *lhs, PyObject *rhs) \ @@ -2442,13 +2451,22 @@ FLOAT_LONG_ACTION(float_compactlong_true_div, /) /* long-float */ -static int +static inline int compactlong_float_guard(PyObject *lhs, PyObject *rhs) { return ( - PyFloat_CheckExact(rhs) && PyLong_CheckExact(lhs) && - _PyLong_IsCompact((PyLongObject *)lhs) + _PyLong_IsCompact((PyLongObject *)lhs) && + PyFloat_CheckExact(rhs) && + !isnan(PyFloat_AsDouble(rhs)) + ); +} + +static inline int +nonzero_compactlong_float_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + compactlong_float_guard(lhs, rhs) && PyFloat_AsDouble(rhs) != 0.0 ); } @@ -2469,14 +2487,14 @@ LONG_FLOAT_ACTION(compactlong_float_true_div, /) static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { [NB_ADD] = {float_compactlong_guard, float_compactlong_add}, [NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract}, - [NB_TRUE_DIVIDE] = {float_compactlong_guard, float_compactlong_true_div}, + [NB_TRUE_DIVIDE] = {nonzero_float_compactlong_guard, float_compactlong_true_div}, [NB_MULTIPLY] = {float_compactlong_guard, float_compactlong_multiply}, }; static _PyBinaryOpSpecializationDescr compactlong_float_specs[NB_OPARG_LAST+1] = { [NB_ADD] = {compactlong_float_guard, compactlong_float_add}, [NB_SUBTRACT] = {compactlong_float_guard, compactlong_float_subtract}, - [NB_TRUE_DIVIDE] = {compactlong_float_guard, compactlong_float_true_div}, + [NB_TRUE_DIVIDE] = {nonzero_compactlong_float_guard, compactlong_float_true_div}, [NB_MULTIPLY] = {compactlong_float_guard, compactlong_float_multiply}, };

1 0

gh-80222: Fix email address header folding with long quoted-string (#122753)
by bitdancer Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/5aaf41685834901e4ed0a40f4c055b9299… commit: 5aaf41685834901e4ed0a40f4c055b92991a0bb5 branch: main author: Mike Edmunds <medmunds(a)gmail.com> committer: bitdancer <rdmurray(a)bitdance.com> date: 2025-01-18T19:50:52-05:00 summary: gh-80222: Fix email address header folding with long quoted-string (#122753) Email generators using email.policy.default could incorrectly omit the quote ('"') characters from a quoted-string during header refolding, leading to invalid address headers and enabling header spoofing. This change restores the quote characters on a bare-quoted-string as the header is refolded, and escapes backslash and quote chars in the string. files: A Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst M Lib/email/_header_value_parser.py M Lib/test/test_email/test__header_value_parser.py diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index ec2215a5e5f33c..3d845c09d415f6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -95,8 +95,16 @@ NLSET = {'\n', '\r'} SPECIALSNL = SPECIALS | NLSET + +def make_quoted_pairs(value): + """Escape dquote and backslash for use within a quoted-string.""" + return str(value).replace('\\', '\\\\').replace('"', '\\"') + + def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' + escaped = make_quoted_pairs(value) + return f'"{escaped}"' + # Match a RFC 2047 word, looks like =?utf-8?q?someword?= rfc2047_matcher = re.compile(r''' @@ -2905,6 +2913,15 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. newparts = list(part) + if part.token_type == 'bare-quoted-string': + # To fold a quoted string we need to create a list of terminal + # tokens that will render the leading and trailing quotes + # and use quoted pairs in the value as appropriate. + newparts = ( + [ValueTerminal('"', 'ptext')] + + [ValueTerminal(make_quoted_pairs(p), 'ptext') + for p in newparts] + + [ValueTerminal('"', 'ptext')]) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 95224e19f67ce5..d60a7039f9d4c6 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3082,13 +3082,40 @@ def test_address_list_with_list_separator_after_fold(self): self._test(parser.get_address_list(to)[0], f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus <beautiful(a)example.com>\n') - a = '.' * 79 + a = '.' * 79 # ('.' is a special, so must be in quoted-string.) to = f'"{a}" <xyz(a)example.com>, "Hübsch Kaktus" <beautiful(a)example.com>' self._test(parser.get_address_list(to)[0], - f'{a}\n' + f'"{a}"\n' ' <xyz(a)example.com>, =?utf-8?q?H=C3=BCbsch?= Kaktus ' '<beautiful(a)example.com>\n') + def test_address_list_with_specials_in_long_quoted_string(self): + # Regression for gh-80222. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('"Exfiltrator <spy(a)example.org> (unclosed comment?" <to(a)example.com>', + '"Exfiltrator <spy(a)example.org> (unclosed\n' + ' comment?" <to(a)example.com>\n'), + ('"Escaped \\" chars \\\\ in quoted-string stay escaped" <to(a)example.com>', + '"Escaped \\" chars \\\\ in quoted-string\n' + ' stay escaped" <to(a)example.com>\n'), + ('This long display name does not need quotes <to(a)example.com>', + 'This long display name does not need\n' + ' quotes <to(a)example.com>\n'), + ('"Quotes are not required but are retained here" <to(a)example.com>', + '"Quotes are not required but are\n' + ' retained here" <to(a)example.com>\n'), + ('"A quoted-string, it can be a valid local-part"@example.com', + '"A quoted-string, it can be a valid\n' + ' local-part"@example.com\n'), + ('"local-part-with-specials(a)but-no-fws.cannot-fold"@example.com', + '"local-part-with-specials(a)but-no-fws.cannot-fold"@example.com\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst new file mode 100644 index 00000000000000..0f0661d0b1cf4a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst @@ -0,0 +1,6 @@ +Fix bug in the folding of quoted strings when flattening an email message using +a modern email policy. Previously when a quoted string was folded so that +it spanned more than one line, the surrounding quotes and internal escapes +would be omitted. This could theoretically be used to spoof header lines +using a carefully constructed quoted string if the resulting rendered email +was transmitted or re-parsed.

1 0

[3.13] gh-128679: Fix tracemalloc.stop() race conditions (#128897)
by vstinner Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/6b47499510e47c0401d1f6cca2660fc12c… commit: 6b47499510e47c0401d1f6cca2660fc12c496e39 branch: 3.13 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-18T23:39:07Z summary: [3.13] gh-128679: Fix tracemalloc.stop() race conditions (#128897) tracemalloc_alloc(), tracemalloc_realloc(), PyTraceMalloc_Track(), PyTraceMalloc_Untrack() and _PyTraceMalloc_TraceRef() now check tracemalloc_config.tracing after calling TABLES_LOCK(). _PyTraceMalloc_Stop() now protects more code with TABLES_LOCK(), especially setting tracemalloc_config.tracing to 1. Add a test using PyTraceMalloc_Track() to test tracemalloc.stop() race condition. Call _PyTraceMalloc_Init() at Python startup. files: A Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst M Include/internal/pycore_tracemalloc.h M Lib/test/test_tracemalloc.py M Modules/_testcapimodule.c M Modules/_tracemalloc.c M Python/pylifecycle.c M Python/tracemalloc.c diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 7ddc5bac5d10af..f70d47074f813c 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -144,7 +144,7 @@ extern PyObject* _PyTraceMalloc_GetTraces(void); extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj); /* Initialize tracemalloc */ -extern int _PyTraceMalloc_Init(void); +extern PyStatus _PyTraceMalloc_Init(void); /* Start tracemalloc */ extern int _PyTraceMalloc_Start(int max_nframe); diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 5755f7697de91a..a848363fcd1de9 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -7,8 +7,9 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support -from test.support import os_helper from test.support import force_not_colorized +from test.support import os_helper +from test.support import threading_helper try: import _testcapi @@ -952,7 +953,6 @@ def check_env_var_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") - def test_env_var_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -1101,6 +1101,14 @@ def test_stop_untrack(self): with self.assertRaises(RuntimeError): self.untrack() + @unittest.skipIf(_testcapi is None, 'need _testcapi') + @threading_helper.requires_working_threading() + # gh-128679: Test crash on a debug build (especially on FreeBSD). + @unittest.skipIf(support.Py_DEBUG, 'need release build') + def test_tracemalloc_track_race(self): + # gh-128679: Test fix for tracemalloc.stop() race condition + _testcapi.tracemalloc_track_race() + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst new file mode 100644 index 00000000000000..837f90df07a705 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst @@ -0,0 +1,3 @@ +Fix :func:`tracemalloc.stop` race condition. Fix :mod:`tracemalloc` to +support calling :func:`tracemalloc.stop` in one thread, while another thread +is tracing memory allocations. Patch by Victor Stinner. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9f1a8faae7e8ed..633c53a26246a8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3322,6 +3322,104 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } + +static void +tracemalloc_track_race_thread(void *data) +{ + PyTraceMalloc_Track(123, 10, 1); + + PyThread_type_lock lock = (PyThread_type_lock)data; + PyThread_release_lock(lock); +} + +// gh-128679: Test fix for tracemalloc.stop() race condition +static PyObject * +tracemalloc_track_race(PyObject *self, PyObject *args) +{ +#define NTHREAD 50 + PyObject *tracemalloc = NULL; + PyObject *stop = NULL; + PyThread_type_lock locks[NTHREAD]; + memset(locks, 0, sizeof(locks)); + + // Call tracemalloc.start() + tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc == NULL) { + goto error; + } + PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); + if (start == NULL) { + goto error; + } + PyObject *res = PyObject_CallNoArgs(start); + Py_DECREF(start); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + stop = PyObject_GetAttrString(tracemalloc, "stop"); + Py_CLEAR(tracemalloc); + if (stop == NULL) { + goto error; + } + + // Start threads + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + PyErr_NoMemory(); + goto error; + } + locks[i] = lock; + PyThread_acquire_lock(lock, 1); + + unsigned long thread; + thread = PyThread_start_new_thread(tracemalloc_track_race_thread, + (void*)lock); + if (thread == (unsigned long)-1) { + PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); + goto error; + } + } + + // Call tracemalloc.stop() while threads are running + res = PyObject_CallNoArgs(stop); + Py_CLEAR(stop); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + // Wait until threads complete with the GIL released + Py_BEGIN_ALLOW_THREADS + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_acquire_lock(lock, 1); + PyThread_release_lock(lock); + } + Py_END_ALLOW_THREADS + + // Free threads locks + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_free_lock(lock); + } + Py_RETURN_NONE; + +error: + Py_CLEAR(tracemalloc); + Py_CLEAR(stop); + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + if (lock) { + PyThread_free_lock(lock); + } + } + return NULL; +#undef NTHREAD +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3464,6 +3562,7 @@ static PyMethodDef TestMethods[] = { {"function_set_warning", function_set_warning, METH_NOARGS}, {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_atexit", test_atexit, METH_NOARGS}, + {"tracemalloc_track_race", tracemalloc_track_race, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 887a1e820e250e..0b85187e5fce07 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -223,10 +223,5 @@ PyInit__tracemalloc(void) PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif - if (_PyTraceMalloc_Init() < 0) { - Py_DECREF(m); - return NULL; - } - return m; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d545123a8a7e22..5255d137ccca89 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -706,6 +706,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_NO_MEMORY(); } + status = _PyTraceMalloc_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + PyThreadState *tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); if (tstate == NULL) { diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index e58b60ddd5e484..d86972b7bc0b97 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() @@ -538,12 +539,16 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (ADD_TRACE(ptr, nelem * elsize) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); - alloc->free(alloc->ctx, ptr); - return NULL; + + if (tracemalloc_config.tracing) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { + /* Failed to allocate a trace for the new memory block */ + alloc->free(alloc->ctx, ptr); + ptr = NULL; + } } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); return ptr; } @@ -559,11 +564,15 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 == NULL) return NULL; + TABLES_LOCK(); + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto done; + } + if (ptr != NULL) { /* an existing memory block has been resized */ - TABLES_LOCK(); - /* tracemalloc_add_trace() updates the trace if there is already a trace at address ptr2 */ if (ptr2 != ptr) { @@ -582,20 +591,19 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) allocating memory. */ Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } - TABLES_UNLOCK(); } else { /* new allocation */ - TABLES_LOCK(); if (ADD_TRACE(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); - return NULL; + ptr2 = NULL; } - TABLES_UNLOCK(); } + +done: + TABLES_UNLOCK(); return ptr2; } @@ -614,7 +622,12 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - REMOVE_TRACE(ptr); + + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); } @@ -673,7 +686,9 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - REMOVE_TRACE(ptr); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } TABLES_UNLOCK(); } return ptr2; @@ -748,7 +763,9 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - REMOVE_TRACE(ptr); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } TABLES_UNLOCK(); } return ptr2; @@ -779,46 +796,36 @@ tracemalloc_clear_filename(void *value) /* reentrant flag must be set to call this function and GIL must be held */ static void -tracemalloc_clear_traces(void) +tracemalloc_clear_traces_unlocked(void) { + set_reentrant(1); + /* The GIL protects variables against concurrent access */ assert(PyGILState_Check()); - TABLES_LOCK(); _Py_hashtable_clear(tracemalloc_traces); _Py_hashtable_clear(tracemalloc_domains); tracemalloc_traced_memory = 0; tracemalloc_peak_traced_memory = 0; - TABLES_UNLOCK(); _Py_hashtable_clear(tracemalloc_tracebacks); _Py_hashtable_clear(tracemalloc_filenames); + + set_reentrant(0); } -int +PyStatus _PyTraceMalloc_Init(void) { - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { - PyErr_SetString(PyExc_RuntimeError, - "the tracemalloc module has been unloaded"); - return -1; - } - - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) - return 0; + assert(tracemalloc_config.initialized == TRACEMALLOC_NOT_INITIALIZED); PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { -#ifdef MS_WINDOWS - PyErr_SetFromWindowsErr(0); -#else - PyErr_SetFromErrno(PyExc_OSError); -#endif - return -1; + return _PyStatus_NO_MEMORY(); } #endif @@ -826,8 +833,7 @@ _PyTraceMalloc_Init(void) if (tables_lock == NULL) { tables_lock = PyThread_allocate_lock(); if (tables_lock == NULL) { - PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); - return -1; + return _PyStatus_NO_MEMORY(); } } #endif @@ -844,9 +850,9 @@ _PyTraceMalloc_Init(void) tracemalloc_domains = tracemalloc_create_domains_table(); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL || tracemalloc_domains == NULL) { - PyErr_NoMemory(); - return -1; + || tracemalloc_traces == NULL || tracemalloc_domains == NULL) + { + return _PyStatus_NO_MEMORY(); } tracemalloc_empty_traceback.nframe = 1; @@ -857,7 +863,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; - return 0; + return _PyStatus_OK(); } @@ -902,10 +908,6 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - if (_PyTraceMalloc_Init() < 0) { - return -1; - } - if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { return -1; } @@ -960,8 +962,13 @@ _PyTraceMalloc_Start(int max_nframe) void _PyTraceMalloc_Stop(void) { - if (!tracemalloc_config.tracing) - return; + // Lock to synchronize with tracemalloc_free() which checks + // 'tracing' while holding the lock. + TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + goto done; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -973,11 +980,14 @@ _PyTraceMalloc_Stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - tracemalloc_clear_traces(); + tracemalloc_clear_traces_unlocked(); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; + +done: + TABLES_UNLOCK(); } @@ -1227,23 +1237,17 @@ tracemalloc_pyobject_decref(void *value) static traceback_t* -tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) +tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) { - - if (!tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) { return NULL; + } - trace_t *trace; - TABLES_LOCK(); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); + trace_t *trace = NULL; if (traces) { trace = _Py_hashtable_get(traces, TO_PTR(ptr)); } - else { - trace = NULL; - } - TABLES_UNLOCK(); - if (!trace) { return NULL; } @@ -1272,13 +1276,20 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + if (tracemalloc_config.tracing) { + traceback = tracemalloc_get_traceback_unlocked(DEFAULT_DOMAIN, + (uintptr_t)ptr); + } + else { + traceback = NULL; PUTS(fd, "Enable tracemalloc to get the memory block " "allocation traceback\n\n"); - return; } - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr); + TABLES_UNLOCK(); + if (traceback == NULL) return; @@ -1307,20 +1318,19 @@ int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { - int res; - PyGILState_STATE gil_state; + PyGILState_STATE gil_state = PyGILState_Ensure(); + TABLES_LOCK(); - if (!tracemalloc_config.tracing) { - /* tracemalloc is not tracing: do nothing */ - return -2; + int res; + if (tracemalloc_config.tracing) { + res = tracemalloc_add_trace(domain, ptr, size); + } + else { + // gh-128679: tracemalloc.stop() was called by another thread + res = -2; } - gil_state = PyGILState_Ensure(); - - TABLES_LOCK(); - res = tracemalloc_add_trace(domain, ptr, size); TABLES_UNLOCK(); - PyGILState_Release(gil_state); return res; } @@ -1329,16 +1339,20 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + int result; + if (tracemalloc_config.tracing) { + tracemalloc_remove_trace(domain, ptr); + result = 0; + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - TABLES_LOCK(); - tracemalloc_remove_trace(domain, ptr); TABLES_UNLOCK(); - - return 0; + return result; } @@ -1376,6 +1390,12 @@ _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ig int res = -1; TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto done; + } + trace_t *trace = _Py_hashtable_get(tracemalloc_traces, TO_PTR(ptr)); if (trace != NULL) { /* update the traceback of the memory block */ @@ -1386,6 +1406,8 @@ _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ig } } /* else: cannot track the object, its memory block size is unknown */ + +done: TABLES_UNLOCK(); return res; @@ -1397,7 +1419,9 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) { traceback_t *traceback; - traceback = tracemalloc_get_traceback(domain, ptr); + TABLES_LOCK(); + traceback = tracemalloc_get_traceback_unlocked(domain, ptr); + TABLES_UNLOCK(); if (traceback == NULL) Py_RETURN_NONE; @@ -1407,19 +1431,20 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) int _PyTraceMalloc_IsTracing(void) { - return tracemalloc_config.tracing; + TABLES_LOCK(); + int tracing = tracemalloc_config.tracing; + TABLES_UNLOCK(); + return tracing; } void _PyTraceMalloc_ClearTraces(void) { - - if (!tracemalloc_config.tracing) { - return; + TABLES_LOCK(); + if (tracemalloc_config.tracing) { + tracemalloc_clear_traces_unlocked(); } - set_reentrant(1); - tracemalloc_clear_traces(); - set_reentrant(0); + TABLES_UNLOCK(); } PyObject * @@ -1506,19 +1531,10 @@ PyObject * _PyTraceMalloc_GetObjectTraceback(PyObject *obj) /*[clinic end generated code: output=41ee0553a658b0aa input=29495f1b21c53212]*/ { - PyTypeObject *type; - traceback_t *traceback; - - type = Py_TYPE(obj); + PyTypeObject *type = Py_TYPE(obj); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)obj - presize); - - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); - if (traceback == NULL) { - Py_RETURN_NONE; - } - - return traceback_to_pyobject(traceback, NULL); + return _PyTraceMalloc_GetTraceback(DEFAULT_DOMAIN, ptr); } int _PyTraceMalloc_GetTracebackLimit(void) { @@ -1530,14 +1546,19 @@ _PyTraceMalloc_GetMemory(void) { size_t size; - size = _Py_hashtable_size(tracemalloc_tracebacks); - size += _Py_hashtable_size(tracemalloc_filenames); - TABLES_LOCK(); - size += _Py_hashtable_size(tracemalloc_traces); - _Py_hashtable_foreach(tracemalloc_domains, - tracemalloc_get_tracemalloc_memory_cb, &size); + if (tracemalloc_config.tracing) { + size = _Py_hashtable_size(tracemalloc_tracebacks); + size += _Py_hashtable_size(tracemalloc_filenames); + size += _Py_hashtable_size(tracemalloc_traces); + _Py_hashtable_foreach(tracemalloc_domains, + tracemalloc_get_tracemalloc_memory_cb, &size); + } + else { + size = 0; + } TABLES_UNLOCK(); + return size; } @@ -1547,12 +1568,15 @@ _PyTraceMalloc_GetTracedMemory(void) { Py_ssize_t size, peak_size; - if (!tracemalloc_config.tracing) - return Py_BuildValue("ii", 0, 0); - TABLES_LOCK(); - size = tracemalloc_traced_memory; - peak_size = tracemalloc_peak_traced_memory; + if (tracemalloc_config.tracing) { + size = tracemalloc_traced_memory; + peak_size = tracemalloc_peak_traced_memory; + } + else { + size = 0; + peak_size = 0; + } TABLES_UNLOCK(); return Py_BuildValue("nn", size, peak_size); @@ -1561,10 +1585,9 @@ _PyTraceMalloc_GetTracedMemory(void) void _PyTraceMalloc_ResetPeak(void) { - if (!tracemalloc_config.tracing) { - return; - } TABLES_LOCK(); - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.tracing) { + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + } TABLES_UNLOCK(); }

1 0

[3.12] gh-128991: Release the enter frame reference within bdb callba… (#129003)
by gaogaotiantian Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/6d638c2655374f2fbd80779aff349324db… commit: 6d638c2655374f2fbd80779aff349324dbc857f8 branch: 3.12 author: Tian Gao <gaogaotiantian(a)hotmail.com> committer: gaogaotiantian <gaogaotiantian(a)hotmail.com> date: 2025-01-18T17:21:34-05:00 summary: [3.12] gh-128991: Release the enter frame reference within bdb callba… (#129003) [3.12] gh-128991: Release the enter frame reference within bdb callback (GH-128992) * Release the enter frame reference within bdb callback * 📜🤖 Added by blurb_it. --------- (cherry picked from commit 61b35f74aa4a6ac606635e245147ff3658628d99) files: A Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst M Lib/bdb.py diff --git a/Lib/bdb.py b/Lib/bdb.py index 3486deacd86a7c..8b63d9eca6da5f 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -60,6 +61,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -84,24 +91,26 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -335,12 +344,12 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() - self.enterframe = None + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + frame = frame.f_back + self.set_step() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -357,7 +366,6 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back - self.enterframe = None def set_quit(self): """Set quitting attribute to True. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst new file mode 100644 index 00000000000000..64fa04fb53e89c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst @@ -0,0 +1 @@ +Release the enter frame reference within :mod:`bdb` callback

1 0

[3.13] gh-128991: Release the enter frame reference within bdb callba… (#129002)
by gaogaotiantian Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/ef9961840b546aba33b168fd8f5788e1ea… commit: ef9961840b546aba33b168fd8f5788e1ea1960e8 branch: 3.13 author: Tian Gao <gaogaotiantian(a)hotmail.com> committer: gaogaotiantian <gaogaotiantian(a)hotmail.com> date: 2025-01-18T17:21:23-05:00 summary: [3.13] gh-128991: Release the enter frame reference within bdb callba… (#129002) [3.13] gh-128991: Release the enter frame reference within bdb callback (GH-128992) * Release the enter frame reference within bdb callback * 📜🤖 Added by blurb_it. --------- (cherry picked from commit 61b35f74aa4a6ac606635e245147ff3658628d99) files: A Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst M Lib/bdb.py diff --git a/Lib/bdb.py b/Lib/bdb.py index 3a4453d95f6596..0a3b6dfbfc6025 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -63,6 +64,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -88,28 +95,27 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - self.enterframe = frame - - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - if event == 'opcode': - return self.dispatch_opcode(frame, arg) - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -373,16 +379,15 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() - self.enterframe = frame - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) - # We need f_trace_lines == True for the debugger to work - frame.f_trace_lines = True - frame = frame.f_back - self.set_stepinstr() - self.enterframe = None + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -402,7 +407,6 @@ def set_continue(self): for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes self.frame_trace_lines_opcodes = {} - self.enterframe = None def set_quit(self): """Set quitting attribute to True. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst new file mode 100644 index 00000000000000..64fa04fb53e89c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst @@ -0,0 +1 @@ +Release the enter frame reference within :mod:`bdb` callback

1 0

gh-128991: Release the enter frame reference within bdb callback (#128992)
by gaogaotiantian Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/61b35f74aa4a6ac606635e245147ff3658… commit: 61b35f74aa4a6ac606635e245147ff3658628d99 branch: main author: Tian Gao <gaogaotiantian(a)hotmail.com> committer: gaogaotiantian <gaogaotiantian(a)hotmail.com> date: 2025-01-18T16:53:06-05:00 summary: gh-128991: Release the enter frame reference within bdb callback (#128992) * Release the enter frame reference within bdb callback * 📜🤖 Added by blurb_it. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst M Lib/bdb.py diff --git a/Lib/bdb.py b/Lib/bdb.py index 73e249621a053b..a741628e32a981 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -4,6 +4,7 @@ import sys import os import weakref +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -65,6 +66,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -90,28 +97,27 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - self.enterframe = frame - - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - if event == 'opcode': - return self.dispatch_opcode(frame, arg) - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -395,16 +401,15 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() - self.enterframe = frame - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) - # We need f_trace_lines == True for the debugger to work - frame.f_trace_lines = True - frame = frame.f_back - self.set_stepinstr() - self.enterframe = None + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -424,7 +429,6 @@ def set_continue(self): for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes self.frame_trace_lines_opcodes = {} - self.enterframe = None def set_quit(self): """Set quitting attribute to True. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst new file mode 100644 index 00000000000000..64fa04fb53e89c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst @@ -0,0 +1 @@ +Release the enter frame reference within :mod:`bdb` callback

1 0

[3.12] gh-128998: Fix indentation of numbered list and literal block (GH-128999) (#129001)
by AA-Turner Jan. 18, 2025

Jan. 18, 2025

https://github.com/python/cpython/commit/ae8c8b7ffd4d50714342b8f011064ba69c… commit: ae8c8b7ffd4d50714342b8f011064ba69c2df358 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2025-01-18T21:00:14Z summary: [3.12] gh-128998: Fix indentation of numbered list and literal block (GH-128999) (#129001) gh-128998: Fix indentation of numbered list and literal block (GH-128999) (cherry picked from commit e8092e5cdcd6707ac0b16d8fb37fa080a88bcc97) Co-authored-by: Rafael Fontenelle <rffontenelle(a)users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/faq/programming.rst diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 7a86701116413b..4b52404671d666 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1906,28 +1906,30 @@ In the standard library code, you will see several common patterns for correctly using identity tests: 1) As recommended by :pep:`8`, an identity test is the preferred way to check -for ``None``. This reads like plain English in code and avoids confusion with -other objects that may have boolean values that evaluate to false. + for ``None``. This reads like plain English in code and avoids confusion + with other objects that may have boolean values that evaluate to false. 2) Detecting optional arguments can be tricky when ``None`` is a valid input -value. In those situations, you can create a singleton sentinel object -guaranteed to be distinct from other objects. For example, here is how -to implement a method that behaves like :meth:`dict.pop`:: + value. In those situations, you can create a singleton sentinel object + guaranteed to be distinct from other objects. For example, here is how + to implement a method that behaves like :meth:`dict.pop`: - _sentinel = object() + .. code-block:: python - def pop(self, key, default=_sentinel): - if key in self: - value = self[key] - del self[key] - return value - if default is _sentinel: - raise KeyError(key) - return default + _sentinel = object() + + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default 3) Container implementations sometimes need to augment equality tests with -identity tests. This prevents the code from being confused by objects such as -``float('NaN')`` that are not equal to themselves. + identity tests. This prevents the code from being confused by objects + such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of :meth:`!collections.abc.Sequence.__contains__`::

1 0