Mailman 3 January 2025 - Python-checkins

Remove unnecessary LIST_TO_TUPLE conversions (GH-126558)
by brandtbucher 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/004f9fd1f22643100aa8163cc9f7bcde7d… commit: 004f9fd1f22643100aa8163cc9f7bcde7df54973 branch: main author: Brandt Bucher <brandtbucher(a)microsoft.com> committer: brandtbucher <brandtbucher(a)gmail.com> date: 2025-01-08T09:00:11-08:00 summary: Remove unnecessary LIST_TO_TUPLE conversions (GH-126558) files: M Lib/test/test_peepholer.py M Python/flowgraph.c diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index c7da151dce3b37..b5b2b350e77a3b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1193,5 +1193,56 @@ def get_insts(lno1, lno2, op1, op2): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_list_to_tuple_get_iter(self): + # for _ in (*foo, *bar) -> for _ in [*foo, *bar] + INTRINSIC_LIST_TO_TUPLE = 6 + insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + expected_insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + self.cfg_optimization_test(insts, expected_insts, consts=[None]) + + def test_list_to_tuple_get_iter_is_safe(self): + a, b = [], [] + for item in (*(items := [0, 1, 2, 3]),): + a.append(item) + b.append(items.pop()) + self.assertEqual(a, [0, 1, 2, 3]) + self.assertEqual(b, [3, 2, 1, 0]) + self.assertEqual(items, []) + + if __name__ == "__main__": unittest.main() diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 017216aadd1f01..24561c1ee04db9 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_flowgraph.h" #include "pycore_compile.h" +#include "pycore_intrinsics.h" #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_opcode_utils.h" @@ -1874,6 +1875,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } break; + case CALL_INTRINSIC_1: + // for _ in (*foo, *bar) -> for _ in [*foo, *bar] + if (oparg == INTRINSIC_LIST_TO_TUPLE && nextop == GET_ITER) { + INSTR_SET_OP0(inst, NOP); + } + break; } }

1 0

gh-111178: fix UBSan failures in `Objects/capsule.c` (GH-128239)
by encukou 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/845d924efb2a144120421260e62b9c4c97… commit: 845d924efb2a144120421260e62b9c4c9726fc69 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-08T14:55:04+01:00 summary: gh-111178: fix UBSan failures in `Objects/capsule.c` (GH-128239) fix UBSan failures for `PyCapsule` files: M Objects/capsule.c diff --git a/Objects/capsule.c b/Objects/capsule.c index 28965e0f21b7a0..16ae65905ef5ac 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -18,6 +18,8 @@ typedef struct { } PyCapsule; +#define _PyCapsule_CAST(op) ((PyCapsule *)(op)) + static int _is_legal_capsule(PyObject *op, const char *invalid_capsule) @@ -284,7 +286,7 @@ PyCapsule_Import(const char *name, int no_block) static void capsule_dealloc(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)op; + PyCapsule *capsule = _PyCapsule_CAST(op); PyObject_GC_UnTrack(op); if (capsule->destructor) { capsule->destructor(op); @@ -296,7 +298,7 @@ capsule_dealloc(PyObject *op) static PyObject * capsule_repr(PyObject *o) { - PyCapsule *capsule = (PyCapsule *)o; + PyCapsule *capsule = _PyCapsule_CAST(o); const char *name; const char *quote; @@ -314,28 +316,27 @@ capsule_repr(PyObject *o) static int -capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) +capsule_traverse(PyObject *self, visitproc visit, void *arg) { // Capsule object is only tracked by the GC // if _PyCapsule_SetTraverse() is called, but // this can still be manually triggered by gc.get_referents() - + PyCapsule *capsule = _PyCapsule_CAST(self); if (capsule->traverse_func != NULL) { - return capsule->traverse_func((PyObject*)capsule, visit, arg); + return capsule->traverse_func(self, visit, arg); } - return 0; } static int -capsule_clear(PyCapsule *capsule) +capsule_clear(PyObject *self) { // Capsule object is only tracked by the GC // if _PyCapsule_SetTraverse() is called + PyCapsule *capsule = _PyCapsule_CAST(self); assert(capsule->clear_func != NULL); - - return capsule->clear_func((PyObject*)capsule); + return capsule->clear_func(self); } @@ -358,8 +359,8 @@ PyTypeObject PyCapsule_Type = { .tp_dealloc = capsule_dealloc, .tp_repr = capsule_repr, .tp_doc = PyCapsule_Type__doc__, - .tp_traverse = (traverseproc)capsule_traverse, - .tp_clear = (inquiry)capsule_clear, + .tp_traverse = capsule_traverse, + .tp_clear = capsule_clear, };

1 0

gh-111178: fix UBSan failures in `Python/context.c` (GH-128242)
by encukou 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/1da0901894d7c0d56ebce97cd0c16aeffb… commit: 1da0901894d7c0d56ebce97cd0c16aeffb64adcf branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-08T14:52:27+01:00 summary: gh-111178: fix UBSan failures in `Python/context.c` (GH-128242) * fix UBSan failures for `PyContext` * fix UBSan failures for `PyContextVar` * fix UBSan failures for `PyContextToken` * fix UBSan failures for `_PyContextTokenMissing` files: M Python/context.c diff --git a/Python/context.c b/Python/context.c index 95aa82206270f9..f30b59b9443bbf 100644 --- a/Python/context.c +++ b/Python/context.c @@ -419,6 +419,9 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=bdf87f8e0cb580e8]*/ +#define _PyContext_CAST(op) ((PyContext *)(op)) + + static inline PyContext * _context_alloc(void) { @@ -513,28 +516,30 @@ context_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -context_tp_clear(PyContext *self) +context_tp_clear(PyObject *op) { + PyContext *self = _PyContext_CAST(op); Py_CLEAR(self->ctx_prev); Py_CLEAR(self->ctx_vars); return 0; } static int -context_tp_traverse(PyContext *self, visitproc visit, void *arg) +context_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContext *self = _PyContext_CAST(op); Py_VISIT(self->ctx_prev); Py_VISIT(self->ctx_vars); return 0; } static void -context_tp_dealloc(PyContext *self) +context_tp_dealloc(PyObject *self) { _PyObject_GC_UNTRACK(self); - - if (self->ctx_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + PyContext *ctx = _PyContext_CAST(self); + if (ctx->ctx_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)context_tp_clear(self); @@ -542,8 +547,9 @@ context_tp_dealloc(PyContext *self) } static PyObject * -context_tp_iter(PyContext *self) +context_tp_iter(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_NewIterKeys(self->ctx_vars); } @@ -575,18 +581,20 @@ context_tp_richcompare(PyObject *v, PyObject *w, int op) } static Py_ssize_t -context_tp_len(PyContext *self) +context_tp_len(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_Len(self->ctx_vars); } static PyObject * -context_tp_subscript(PyContext *self, PyObject *key) +context_tp_subscript(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return NULL; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); int found = _PyHamt_Find(self->ctx_vars, key, &val); if (found < 0) { return NULL; @@ -599,12 +607,13 @@ context_tp_subscript(PyContext *self, PyObject *key) } static int -context_tp_contains(PyContext *self, PyObject *key) +context_tp_contains(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return -1; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); return _PyHamt_Find(self->ctx_vars, key, &val); } @@ -701,7 +710,7 @@ _contextvars_Context_copy_impl(PyContext *self) static PyObject * -context_run(PyContext *self, PyObject *const *args, +context_run(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyThreadState *ts = _PyThreadState_GET(); @@ -712,14 +721,14 @@ context_run(PyContext *self, PyObject *const *args, return NULL; } - if (_PyContext_Enter(ts, (PyObject *)self)) { + if (_PyContext_Enter(ts, self)) { return NULL; } PyObject *call_result = _PyObject_VectorcallTstate( ts, args[0], args + 1, nargs - 1, kwnames); - if (_PyContext_Exit(ts, (PyObject *)self)) { + if (_PyContext_Exit(ts, self)) { Py_XDECREF(call_result); return NULL; } @@ -739,21 +748,12 @@ static PyMethodDef PyContext_methods[] = { }; static PySequenceMethods PyContext_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)context_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = context_tp_contains }; static PyMappingMethods PyContext_as_mapping = { - (lenfunc)context_tp_len, /* mp_length */ - (binaryfunc)context_tp_subscript, /* mp_subscript */ + .mp_length = context_tp_len, + .mp_subscript = context_tp_subscript }; PyTypeObject PyContext_Type = { @@ -763,13 +763,13 @@ PyTypeObject PyContext_Type = { .tp_methods = PyContext_methods, .tp_as_mapping = &PyContext_as_mapping, .tp_as_sequence = &PyContext_as_sequence, - .tp_iter = (getiterfunc)context_tp_iter, - .tp_dealloc = (destructor)context_tp_dealloc, + .tp_iter = context_tp_iter, + .tp_dealloc = context_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = context_tp_richcompare, - .tp_traverse = (traverseproc)context_tp_traverse, - .tp_clear = (inquiry)context_tp_clear, + .tp_traverse = context_tp_traverse, + .tp_clear = context_tp_clear, .tp_new = context_tp_new, .tp_weaklistoffset = offsetof(PyContext, ctx_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -909,6 +909,9 @@ class _contextvars.ContextVar "PyContextVar *" "&PyContextVar_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=445da935fa8883c3]*/ +#define _PyContextVar_CAST(op) ((PyContextVar *)(op)) + + static PyObject * contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -926,8 +929,9 @@ contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -contextvar_tp_clear(PyContextVar *self) +contextvar_tp_clear(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); Py_CLEAR(self->var_name); Py_CLEAR(self->var_default); #ifndef Py_GIL_DISABLED @@ -939,15 +943,16 @@ contextvar_tp_clear(PyContextVar *self) } static int -contextvar_tp_traverse(PyContextVar *self, visitproc visit, void *arg) +contextvar_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextVar *self = _PyContextVar_CAST(op); Py_VISIT(self->var_name); Py_VISIT(self->var_default); return 0; } static void -contextvar_tp_dealloc(PyContextVar *self) +contextvar_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)contextvar_tp_clear(self); @@ -955,14 +960,16 @@ contextvar_tp_dealloc(PyContextVar *self) } static Py_hash_t -contextvar_tp_hash(PyContextVar *self) +contextvar_tp_hash(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); return self->var_hash; } static PyObject * -contextvar_tp_repr(PyContextVar *self) +contextvar_tp_repr(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); // Estimation based on the shortest name and default value, // but maximize the pointer size. // "<ContextVar name='a' at 0x1234567812345678>" @@ -1106,15 +1113,15 @@ PyTypeObject PyContextVar_Type = { sizeof(PyContextVar), .tp_methods = PyContextVar_methods, .tp_members = PyContextVar_members, - .tp_dealloc = (destructor)contextvar_tp_dealloc, + .tp_dealloc = contextvar_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)contextvar_tp_traverse, - .tp_clear = (inquiry)contextvar_tp_clear, + .tp_traverse = contextvar_tp_traverse, + .tp_clear = contextvar_tp_clear, .tp_new = contextvar_tp_new, .tp_free = PyObject_GC_Del, - .tp_hash = (hashfunc)contextvar_tp_hash, - .tp_repr = (reprfunc)contextvar_tp_repr, + .tp_hash = contextvar_tp_hash, + .tp_repr = contextvar_tp_repr, }; @@ -1129,6 +1136,9 @@ class _contextvars.Token "PyContextToken *" "&PyContextToken_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=338a5e2db13d3f5b]*/ +#define _PyContextToken_CAST(op) ((PyContextToken *)(op)) + + static PyObject * token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1138,8 +1148,9 @@ token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -token_tp_clear(PyContextToken *self) +token_tp_clear(PyObject *op) { + PyContextToken *self = _PyContextToken_CAST(op); Py_CLEAR(self->tok_ctx); Py_CLEAR(self->tok_var); Py_CLEAR(self->tok_oldval); @@ -1147,8 +1158,9 @@ token_tp_clear(PyContextToken *self) } static int -token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) +token_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextToken *self = _PyContextToken_CAST(op); Py_VISIT(self->tok_ctx); Py_VISIT(self->tok_var); Py_VISIT(self->tok_oldval); @@ -1156,7 +1168,7 @@ token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) } static void -token_tp_dealloc(PyContextToken *self) +token_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)token_tp_clear(self); @@ -1164,8 +1176,9 @@ token_tp_dealloc(PyContextToken *self) } static PyObject * -token_tp_repr(PyContextToken *self) +token_tp_repr(PyObject *op) { + PyContextToken *self = _PyContextToken_CAST(op); PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); if (writer == NULL) { return NULL; @@ -1195,14 +1208,16 @@ token_tp_repr(PyContextToken *self) } static PyObject * -token_get_var(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_var(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); return Py_NewRef(self->tok_var);; } static PyObject * -token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_old_value(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); if (self->tok_oldval == NULL) { return get_token_missing(); } @@ -1211,8 +1226,8 @@ token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) } static PyGetSetDef PyContextTokenType_getsetlist[] = { - {"var", (getter)token_get_var, NULL, NULL}, - {"old_value", (getter)token_get_old_value, NULL, NULL}, + {"var", token_get_var, NULL, NULL}, + {"old_value", token_get_old_value, NULL, NULL}, {NULL} }; @@ -1228,15 +1243,15 @@ PyTypeObject PyContextToken_Type = { sizeof(PyContextToken), .tp_methods = PyContextTokenType_methods, .tp_getset = PyContextTokenType_getsetlist, - .tp_dealloc = (destructor)token_tp_dealloc, + .tp_dealloc = token_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)token_tp_traverse, - .tp_clear = (inquiry)token_tp_clear, + .tp_traverse = token_tp_traverse, + .tp_clear = token_tp_clear, .tp_new = token_tp_new, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, - .tp_repr = (reprfunc)token_tp_repr, + .tp_repr = token_tp_repr, }; static PyContextToken * @@ -1270,7 +1285,7 @@ context_token_missing_tp_repr(PyObject *self) } static void -context_token_missing_tp_dealloc(_PyContextTokenMissing *Py_UNUSED(self)) +context_token_missing_tp_dealloc(PyObject *Py_UNUSED(self)) { #ifdef Py_DEBUG /* The singleton is statically allocated. */ @@ -1285,7 +1300,7 @@ PyTypeObject _PyContextTokenMissing_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "Token.MISSING", sizeof(_PyContextTokenMissing), - .tp_dealloc = (destructor)context_token_missing_tp_dealloc, + .tp_dealloc = context_token_missing_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_repr = context_token_missing_tp_repr,

1 0

gh-111178: fix UBSan failures in `Python/hamt.c` (GH-128247)
by encukou 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/c22302ecea8c22379a41cd77158748d5fc… commit: c22302ecea8c22379a41cd77158748d5fcd91352 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-08T14:50:40+01:00 summary: gh-111178: fix UBSan failures in `Python/hamt.c` (GH-128247) * fix UBSan failures for `PyHamtObject` * fix UBSan failures for `PyHamtNode_Array` * fix UBSan failures for `PyHamtNode_Collision` * fix UBSan failures for `PyHamtNode_Bitmap` files: M Python/hamt.c diff --git a/Python/hamt.c b/Python/hamt.c index cfd211f4541446..ed43a0449d7a01 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -319,6 +319,8 @@ typedef struct { Py_ssize_t a_count; } PyHamtNode_Array; +#define _PyHamtNode_Array_CAST(op) ((PyHamtNode_Array *)(op)) + typedef struct { PyObject_VAR_HEAD @@ -326,6 +328,8 @@ typedef struct { PyObject *c_array[1]; } PyHamtNode_Collision; +#define _PyHamtNode_Collision_CAST(op) ((PyHamtNode_Collision *)(op)) + static PyHamtObject * hamt_alloc(void); @@ -479,6 +483,8 @@ _hamt_dump_ident(PyUnicodeWriter *writer, int level) #endif /* Py_DEBUG */ /////////////////////////////////// Bitmap Node +#define _PyHamtNode_Bitmap_CAST(op) ((PyHamtNode_Bitmap *)(op)) + static PyHamtNode * hamt_node_bitmap_new(Py_ssize_t size) @@ -1083,30 +1089,27 @@ hamt_node_bitmap_find(PyHamtNode_Bitmap *self, } static int -hamt_node_bitmap_traverse(PyHamtNode_Bitmap *self, visitproc visit, void *arg) +hamt_node_bitmap_traverse(PyObject *op, visitproc visit, void *arg) { /* Bitmap's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Bitmap *self = _PyHamtNode_Bitmap_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0;) { Py_VISIT(self->b_array[i]); } - return 0; } static void -hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) +hamt_node_bitmap_dealloc(PyObject *self) { /* Bitmap's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - Py_ssize_t i; + PyHamtNode_Bitmap *node = _PyHamtNode_Bitmap_CAST(self); + Py_ssize_t i, len = Py_SIZE(self); - if (Py_SIZE(self) == 0) { + if (len == 0) { /* The empty node is statically allocated. */ - assert(self == &_Py_SINGLETON(hamt_bitmap_node_empty)); + assert(node == &_Py_SINGLETON(hamt_bitmap_node_empty)); #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt node bitmap singleton"); #else @@ -1120,11 +1123,11 @@ hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) if (len > 0) { i = len; while (--i >= 0) { - Py_XDECREF(self->b_array[i]); + Py_XDECREF(node->b_array[i]); } } - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -1489,38 +1492,30 @@ hamt_node_collision_find(PyHamtNode_Collision *self, static int -hamt_node_collision_traverse(PyHamtNode_Collision *self, - visitproc visit, void *arg) +hamt_node_collision_traverse(PyObject *op, visitproc visit, void *arg) { /* Collision's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Collision *self = _PyHamtNode_Collision_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0; ) { Py_VISIT(self->c_array[i]); } - return 0; } static void -hamt_node_collision_dealloc(PyHamtNode_Collision *self) +hamt_node_collision_dealloc(PyObject *self) { /* Collision's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_collision_dealloc) - if (len > 0) { - + PyHamtNode_Collision *node = _PyHamtNode_Collision_CAST(self); while (--len >= 0) { - Py_XDECREF(self->c_array[len]); + Py_XDECREF(node->c_array[len]); } } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -1868,35 +1863,27 @@ hamt_node_array_find(PyHamtNode_Array *self, } static int -hamt_node_array_traverse(PyHamtNode_Array *self, - visitproc visit, void *arg) +hamt_node_array_traverse(PyObject *op, visitproc visit, void *arg) { /* Array's tp_traverse */ - - Py_ssize_t i; - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + PyHamtNode_Array *self = _PyHamtNode_Array_CAST(op); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { Py_VISIT(self->a_array[i]); } - return 0; } static void -hamt_node_array_dealloc(PyHamtNode_Array *self) +hamt_node_array_dealloc(PyObject *self) { /* Array's tp_dealloc */ - - Py_ssize_t i; - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_array_dealloc) - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { - Py_XDECREF(self->a_array[i]); + PyHamtNode_Array *obj = _PyHamtNode_Array_CAST(self); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + Py_XDECREF(obj->a_array[i]); } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } @@ -2605,6 +2592,8 @@ static PyObject * hamt_dump(PyHamtObject *self); #endif +#define _PyHamtObject_CAST(op) ((PyHamtObject *)(op)) + static PyObject * hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2613,24 +2602,27 @@ hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -hamt_tp_clear(PyHamtObject *self) +hamt_tp_clear(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_CLEAR(self->h_root); return 0; } static int -hamt_tp_traverse(PyHamtObject *self, visitproc visit, void *arg) +hamt_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_VISIT(self->h_root); return 0; } static void -hamt_tp_dealloc(PyHamtObject *self) +hamt_tp_dealloc(PyObject *self) { - if (self == _empty_hamt) { + PyHamtObject *obj = _PyHamtObject_CAST(self); + if (obj == _empty_hamt) { /* The empty one is statically allocated. */ #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt singleton"); @@ -2640,8 +2632,8 @@ hamt_tp_dealloc(PyHamtObject *self) } PyObject_GC_UnTrack(self); - if (self->h_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + if (obj->h_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)hamt_tp_clear(self); Py_TYPE(self)->tp_free(self); @@ -2673,16 +2665,18 @@ hamt_tp_richcompare(PyObject *v, PyObject *w, int op) } static int -hamt_tp_contains(PyHamtObject *self, PyObject *key) +hamt_tp_contains(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Find(self, key, &val); } static PyObject * -hamt_tp_subscript(PyHamtObject *self, PyObject *key) +hamt_tp_subscript(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2698,19 +2692,21 @@ hamt_tp_subscript(PyHamtObject *self, PyObject *key) } static Py_ssize_t -hamt_tp_len(PyHamtObject *self) +hamt_tp_len(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Len(self); } static PyObject * -hamt_tp_iter(PyHamtObject *self) +hamt_tp_iter(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } static PyObject * -hamt_py_set(PyHamtObject *self, PyObject *args) +hamt_py_set(PyObject *op, PyObject *args) { PyObject *key; PyObject *val; @@ -2719,11 +2715,12 @@ hamt_py_set(PyHamtObject *self, PyObject *args) return NULL; } + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Assoc(self, key, val); } static PyObject * -hamt_py_get(PyHamtObject *self, PyObject *args) +hamt_py_get(PyObject *op, PyObject *args) { PyObject *key; PyObject *def = NULL; @@ -2733,6 +2730,7 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } PyObject *val = NULL; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2750,67 +2748,63 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } static PyObject * -hamt_py_delete(PyHamtObject *self, PyObject *key) +hamt_py_delete(PyObject *op, PyObject *key) { + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Without(self, key); } static PyObject * -hamt_py_items(PyHamtObject *self, PyObject *args) +hamt_py_items(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterItems(self); } static PyObject * -hamt_py_values(PyHamtObject *self, PyObject *args) +hamt_py_values(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterValues(self); } static PyObject * -hamt_py_keys(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_keys(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } #ifdef Py_DEBUG static PyObject * -hamt_py_dump(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_dump(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return hamt_dump(self); } #endif static PyMethodDef PyHamt_methods[] = { - {"set", _PyCFunction_CAST(hamt_py_set), METH_VARARGS, NULL}, - {"get", _PyCFunction_CAST(hamt_py_get), METH_VARARGS, NULL}, - {"delete", _PyCFunction_CAST(hamt_py_delete), METH_O, NULL}, - {"items", _PyCFunction_CAST(hamt_py_items), METH_NOARGS, NULL}, - {"keys", _PyCFunction_CAST(hamt_py_keys), METH_NOARGS, NULL}, - {"values", _PyCFunction_CAST(hamt_py_values), METH_NOARGS, NULL}, + {"set", hamt_py_set, METH_VARARGS, NULL}, + {"get", hamt_py_get, METH_VARARGS, NULL}, + {"delete", hamt_py_delete, METH_O, NULL}, + {"items", hamt_py_items, METH_NOARGS, NULL}, + {"keys", hamt_py_keys, METH_NOARGS, NULL}, + {"values", hamt_py_values, METH_NOARGS, NULL}, #ifdef Py_DEBUG - {"__dump__", _PyCFunction_CAST(hamt_py_dump), METH_NOARGS, NULL}, + {"__dump__", hamt_py_dump, METH_NOARGS, NULL}, #endif {NULL, NULL} }; static PySequenceMethods PyHamt_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)hamt_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = hamt_tp_contains, }; static PyMappingMethods PyHamt_as_mapping = { - (lenfunc)hamt_tp_len, /* mp_length */ - (binaryfunc)hamt_tp_subscript, /* mp_subscript */ + .mp_length = hamt_tp_len, + .mp_subscript = hamt_tp_subscript, }; PyTypeObject _PyHamt_Type = { @@ -2820,13 +2814,13 @@ PyTypeObject _PyHamt_Type = { .tp_methods = PyHamt_methods, .tp_as_mapping = &PyHamt_as_mapping, .tp_as_sequence = &PyHamt_as_sequence, - .tp_iter = (getiterfunc)hamt_tp_iter, - .tp_dealloc = (destructor)hamt_tp_dealloc, + .tp_iter = hamt_tp_iter, + .tp_dealloc = hamt_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = hamt_tp_richcompare, - .tp_traverse = (traverseproc)hamt_tp_traverse, - .tp_clear = (inquiry)hamt_tp_clear, + .tp_traverse = hamt_tp_traverse, + .tp_clear = hamt_tp_clear, .tp_new = hamt_tp_new, .tp_weaklistoffset = offsetof(PyHamtObject, h_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -2841,10 +2835,10 @@ PyTypeObject _PyHamt_ArrayNode_Type = { "hamt_array_node", sizeof(PyHamtNode_Array), 0, - .tp_dealloc = (destructor)hamt_node_array_dealloc, + .tp_dealloc = hamt_node_array_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_array_traverse, + .tp_traverse = hamt_node_array_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2854,10 +2848,10 @@ PyTypeObject _PyHamt_BitmapNode_Type = { "hamt_bitmap_node", sizeof(PyHamtNode_Bitmap) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_bitmap_dealloc, + .tp_dealloc = hamt_node_bitmap_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_bitmap_traverse, + .tp_traverse = hamt_node_bitmap_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2867,10 +2861,10 @@ PyTypeObject _PyHamt_CollisionNode_Type = { "hamt_collision_node", sizeof(PyHamtNode_Collision) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_collision_dealloc, + .tp_dealloc = hamt_node_collision_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_collision_traverse, + .tp_traverse = hamt_node_collision_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, };

1 0

[3.13] gh-128552: fix refcycles in eager task creation (#128553) (#128585)
by kumaraditya303 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/13835888e697e7412b905a55d78755ed7a… commit: 13835888e697e7412b905a55d78755ed7a132c5c branch: 3.13 author: Thomas Grainger <tagrain(a)gmail.com> committer: kumaraditya303 <kumaraditya(a)python.org> date: 2025-01-08T18:16:43+05:30 summary: [3.13] gh-128552: fix refcycles in eager task creation (#128553) (#128585) gh-128552: fix refcycles in eager task creation (#128553) (cherry picked from commit 61b9811ac6843e22b5896ef96030d421b79cd892) files: A Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst M Lib/asyncio/base_events.py M Lib/asyncio/taskgroups.py M Lib/test/test_asyncio/test_taskgroups.py diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 91434042685239..ca29792dc89825 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -477,7 +477,12 @@ def create_task(self, coro, *, name=None, context=None): task.set_name(name) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 9fa772ca9d02cc..8af199d6dcc41a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -205,7 +205,12 @@ def create_task(self, coro, *, name=None, context=None): else: self._tasks.add(task) task.add_done_callback(self._on_task_done) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task # Since Python 3.8 Tasks propagate all exceptions correctly, # except for KeyboardInterrupt and SystemExit which are diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 138f59ebf57ef7..e52061aac923cc 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,6 +1,8 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. +import weakref +import sys import gc import asyncio import contextvars @@ -28,7 +30,25 @@ def get_error_types(eg): return {type(exc) for exc in eg.exceptions} -class TestTaskGroup(unittest.IsolatedAsyncioTestCase): +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +(a)contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: async def test_taskgroup_01(self): @@ -822,15 +842,15 @@ async def test_taskgroup_without_parent_task(self): with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - def test_coro_closed_when_tg_closed(self): + async def test_coro_closed_when_tg_closed(self): async def run_coro_after_tg_closes(): async with taskgroups.TaskGroup() as tg: pass coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "is finished"): tg.create_task(coro) - loop = asyncio.get_event_loop() - loop.run_until_complete(run_coro_after_tg_closes()) + + await run_coro_after_tg_closes() async def test_cancelling_level_preserved(self): async def raise_after(t, e): @@ -955,6 +975,30 @@ async def coro_fn(): self.assertIsInstance(exc, _Done) self.assertListEqual(gc.get_referrers(exc), []) + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), []) + async def test_exception_refcycles_propagate_cancellation_error(self): """Test that TaskGroup deletes propagate_cancellation_error""" tg = asyncio.TaskGroup() @@ -988,5 +1032,16 @@ class MyKeyboardInterrupt(KeyboardInterrupt): self.assertListEqual(gc.get_referrers(exc), []) +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst new file mode 100644 index 00000000000000..83816f775da9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst @@ -0,0 +1 @@ +Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and :meth:`asyncio.TaskGroup.create_task` holding a reference to the created task if it is eager.

1 0

[3.13] gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (GH-128616) (#128625)
by sobolevn 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/7e099c51b6404d8d6a5282ac20186d6bd4… commit: 7e099c51b6404d8d6a5282ac20186d6bd4cda17d branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: sobolevn <mail(a)sobolevn.me> date: 2025-01-08T12:32:32Z summary: [3.13] gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (GH-128616) (#128625) gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (GH-128616) (cherry picked from commit 74a517181a9bb65a1f6da149af7427a9fcb3add3) Co-authored-by: sobolevn <mail(a)sobolevn.me> files: M Lib/test/test_typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index cf0406faf11670..140ceb74735570 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5177,6 +5177,18 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ...

1 0

[3.13] gh-128613: Increase `typing.Concatenate` coverage (GH-128614) (#128623)
by AlexWaygood 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/77b411bd90b154ad84158515f0b346b647… commit: 77b411bd90b154ad84158515f0b346b647379ae2 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AlexWaygood <Alex.Waygood(a)Gmail.com> date: 2025-01-08T12:26:01Z summary: [3.13] gh-128613: Increase `typing.Concatenate` coverage (GH-128614) (#128623) Co-authored-by: sobolevn <mail(a)sobolevn.me> files: M Lib/test/test_typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f32bb31280a8ef..cf0406faf11670 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -9998,6 +9998,18 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (Concatenate[int, T, P], T)) self.assertEqual(C4.__parameters__, (T, P)) + def test_invalid_uses(self): + with self.assertRaisesRegex(TypeError, 'Concatenate of no types'): + Concatenate[()] + with self.assertRaisesRegex( + TypeError, + ( + 'The last parameter to Concatenate should be a ' + 'ParamSpec variable or ellipsis' + ), + ): + Concatenate[int] + def test_var_substitution(self): T = TypeVar('T') P = ParamSpec('P')

1 0

[3.12] gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (GH-128616) (#128626)
by AlexWaygood 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/4c130c9238d0bd8d2641f1d2ec667641d8… commit: 4c130c9238d0bd8d2641f1d2ec667641d8de2abd branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AlexWaygood <Alex.Waygood(a)Gmail.com> date: 2025-01-08T12:25:59Z summary: [3.12] gh-128615: Cover pickling of `ParamSpecArgs` and `ParamSpecKwargs` (GH-128616) (#128626) Co-authored-by: sobolevn <mail(a)sobolevn.me> files: M Lib/test/test_typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c1057dc3887378..c5f4d775f22fb6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4838,6 +4838,18 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ...

1 0

[3.12] gh-41872: Fix quick extraction of module docstrings from a file in pydoc (GH-127520) (GH-128621)
by serhiy-storchaka 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/b69b9da9b5ccf11530651b47ff794ced90… commit: b69b9da9b5ccf11530651b47ff794ced904bc65d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2025-01-08T14:21:30+02:00 summary: [3.12] gh-41872: Fix quick extraction of module docstrings from a file in pydoc (GH-127520) (GH-128621) It now supports docstrings with single quotes, escape sequences, raw string literals, and other Python syntax. (cherry picked from commit 474e419792484d1c16e7d9c99b7bf144136b9307) Co-authored-by: Srinivas Reddy Thatiparthy (తాటిపర్తి శ్రీనివాస్ రెడ్డి) <thatiparthysreenivas(a)gmail.com> Co-authored-by: Éric <merwok(a)netwok.org> Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst M Lib/pydoc.py M Lib/test/test_pydoc/test_pydoc.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index e3745e5453b5c3..9dfa87b2a82395 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -54,6 +54,7 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast import __future__ import builtins import importlib._bootstrap @@ -346,21 +347,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 6e80b54335b0ee..8261c8561a1d69 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -3,6 +3,7 @@ import contextlib import importlib.util import inspect +import io import pydoc import py_compile import keyword @@ -841,6 +842,82 @@ def test_synopsis(self): synopsis = pydoc.synopsis(TESTFN, {}) self.assertEqual(synopsis, 'line 1: h\xe9') + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + @requires_docstrings def test_synopsis_sourceless(self): os = import_helper.import_fresh_module('os') diff --git a/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst new file mode 100644 index 00000000000000..b807dcb284c248 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst @@ -0,0 +1,3 @@ +Fix quick extraction of module docstrings from a file in :mod:`pydoc`. +It now supports docstrings with single quotes, escape sequences, +raw string literals, and other Python syntax.

1 0

[3.13] gh-41872: Fix quick extraction of module docstrings from a file in pydoc (GH-127520) (GH-128620)
by serhiy-storchaka 08 Jan '25

08 Jan '25

https://github.com/python/cpython/commit/dda55ccf1a4534e326f76a598b72c28498… commit: dda55ccf1a4534e326f76a598b72c2849873751a branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2025-01-08T14:21:15+02:00 summary: [3.13] gh-41872: Fix quick extraction of module docstrings from a file in pydoc (GH-127520) (GH-128620) It now supports docstrings with single quotes, escape sequences, raw string literals, and other Python syntax. (cherry picked from commit 474e419792484d1c16e7d9c99b7bf144136b9307) Co-authored-by: Srinivas Reddy Thatiparthy (తాటిపర్తి శ్రీనివాస్ రెడ్డి) <thatiparthysreenivas(a)gmail.com> Co-authored-by: Éric <merwok(a)netwok.org> Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst M Lib/pydoc.py M Lib/test/test_pydoc/test_pydoc.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 30cd0b3fa3cb4e..1a527b2c307b68 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -54,6 +54,7 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast import __future__ import builtins import importlib._bootstrap @@ -381,21 +382,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 42776562fb7fa5..9cc2252e29367d 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -4,6 +4,7 @@ import contextlib import importlib.util import inspect +import io import pydoc import py_compile import keyword @@ -879,6 +880,82 @@ def test_synopsis(self): synopsis = pydoc.synopsis(TESTFN, {}) self.assertEqual(synopsis, 'line 1: h\xe9') + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + @requires_docstrings def test_synopsis_sourceless(self): os = import_helper.import_fresh_module('os') diff --git a/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst new file mode 100644 index 00000000000000..b807dcb284c248 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst @@ -0,0 +1,3 @@ +Fix quick extraction of module docstrings from a file in :mod:`pydoc`. +It now supports docstrings with single quotes, escape sequences, +raw string literals, and other Python syntax.

1 0