Mailman 3 January 2025 - Python-checkins

gh-128863: Deprecate _PyLong_FromDigits() function (#127939)
by vstinner Jan. 24, 2025

Jan. 24, 2025

https://github.com/python/cpython/commit/233fd00f0a19d33932e35f2fb6794ecae7… commit: 233fd00f0a19d33932e35f2fb6794ecae745b780 branch: main author: Sergey B Kirpichev <skirpichev(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-24T13:17:10+01:00 summary: gh-128863: Deprecate _PyLong_FromDigits() function (#127939) files: M Doc/deprecations/c-api-pending-removal-in-3.18.rst M Doc/whatsnew/3.14.rst M Include/cpython/longintrepr.h M Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst M Objects/longobject.c diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 361e1a9abf22d7..0689d8b4f9e959 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -7,7 +7,8 @@ Pending removal in Python 3.18 * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f463ed415d6a20..b3c53b78903c7f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1396,7 +1396,8 @@ Deprecated * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f0d3d0d2b37e4f..4dd82600d562ee 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -105,7 +105,7 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); // Return a copy of src. PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); -PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( +Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( int negative, Py_ssize_t digit_count, digit *digits); diff --git a/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst index 7e6a8484b8887c..b2e5664138fba2 100644 --- a/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst +++ b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst @@ -5,7 +5,8 @@ Python 3.18: * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. -* :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. +* :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Objects/longobject.c b/Objects/longobject.c index b4e3a70adf2b5b..905c4695f60d4f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -205,7 +205,6 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) } PyLongObject *result = long_alloc(digit_count); if (result == NULL) { - PyErr_NoMemory(); return NULL; } _PyLong_SetSignAndDigitCount(result, negative?-1:1, digit_count); @@ -217,15 +216,29 @@ PyObject * _PyLong_Copy(PyLongObject *src) { assert(src != NULL); + int sign; if (_PyLong_IsCompact(src)) { stwodigits ival = medium_value(src); if (IS_SMALL_INT(ival)) { return get_small_int((sdigit)ival); } + sign = _PyLong_CompactSign(src); + } + else { + sign = _PyLong_NonCompactSign(src); } + Py_ssize_t size = _PyLong_DigitCount(src); - return (PyObject *)_PyLong_FromDigits(_PyLong_IsNegative(src), size, src->long_value.ob_digit); + PyLongObject *result = long_alloc(size); + + if (result == NULL) { + return NULL; + } + _PyLong_SetSignAndDigitCount(result, sign, size); + memcpy(result->long_value.ob_digit, src->long_value.ob_digit, + size * sizeof(digit)); + return (PyObject *)result; } static PyObject *

1 0

gh-127937: Convert decimal to use PEP 757 import API (#127925)
by vstinner Jan. 24, 2025

Jan. 24, 2025

https://github.com/python/cpython/commit/3d8fc8b9ae5beec852acf1a0e8102da030… commit: 3d8fc8b9ae5beec852acf1a0e8102da030eeb1aa branch: main author: Sergey B Kirpichev <skirpichev(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-24T11:05:52Z summary: gh-127937: Convert decimal to use PEP 757 import API (#127925) Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst M Modules/_decimal/_decimal.c diff --git a/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst b/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst new file mode 100644 index 00000000000000..6cf5fd2872cd43 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst @@ -0,0 +1,3 @@ +Convert the :mod:`decimal` module to use :pep:`757` C API (export-import +integers), offering some speed-up if the integer part of the +:class:`~decimal.Decimal` instance is small. Patch by Sergey B Kirpichev. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0def463c7d8b9e..b9abd8bd2e7a53 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -2336,15 +2336,16 @@ dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, } if (export_long.digits) { const PyLongLayout *layout = PyLong_GetNativeLayout(); - uint32_t base = (uint32_t)1 << layout->bits_per_digit; - uint8_t sign = export_long.negative ? MPD_NEG : MPD_POS; - Py_ssize_t len = export_long.ndigits; - assert(layout->bits_per_digit <= 32); + assert(layout->bits_per_digit < 32); assert(layout->digits_order == -1); assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); assert(layout->digit_size == 2 || layout->digit_size == 4); + uint32_t base = (uint32_t)1 << layout->bits_per_digit; + uint8_t sign = export_long.negative ? MPD_NEG : MPD_POS; + Py_ssize_t len = export_long.ndigits; + if (layout->digit_size == 4) { mpd_qimport_u32(MPD(dec), export_long.digits, len, sign, base, ctx, status); @@ -3642,13 +3643,6 @@ dec_format(PyObject *dec, PyObject *args) static PyObject * dec_as_long(PyObject *dec, PyObject *context, int round) { - PyLongObject *pylong; - digit *ob_digit; - size_t n; - mpd_t *x; - mpd_context_t workctx; - uint32_t status = 0; - if (mpd_isspecial(MPD(dec))) { if (mpd_isnan(MPD(dec))) { PyErr_SetString(PyExc_ValueError, @@ -3661,12 +3655,16 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return NULL; } - x = mpd_qnew(); + mpd_t *x = mpd_qnew(); + if (x == NULL) { PyErr_NoMemory(); return NULL; } - workctx = *CTX(context); + + mpd_context_t workctx = *CTX(context); + uint32_t status = 0; + workctx.round = round; mpd_qround_to_int(x, MPD(dec), &workctx, &status); if (dec_addstatus(context, status)) { @@ -3675,34 +3673,56 @@ dec_as_long(PyObject *dec, PyObject *context, int round) } status = 0; - ob_digit = NULL; -#if PYLONG_BITS_IN_DIGIT == 30 - n = mpd_qexport_u32(&ob_digit, 0, PyLong_BASE, x, &status); -#elif PYLONG_BITS_IN_DIGIT == 15 - n = mpd_qexport_u16(&ob_digit, 0, PyLong_BASE, x, &status); -#else - #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" -#endif + int64_t val = mpd_qget_i64(x, &status); + + if (!status) { + mpd_del(x); + return PyLong_FromInt64(val); + } + assert(!mpd_iszero(x)); + + const PyLongLayout *layout = PyLong_GetNativeLayout(); + + assert(layout->bits_per_digit < 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + + uint32_t base = (uint32_t)1 << layout->bits_per_digit; + /* We use a temporary buffer for digits for now, as for nonzero rdata + mpd_qexport_u32/u16() require either space "allocated by one of + libmpdec’s allocation functions" or "rlen MUST be correct" (to avoid + reallocation). This can be further optimized by using rlen from + mpd_sizeinbase(). See gh-127925. */ + void *tmp_digits = NULL; + size_t n; + + status = 0; + if (layout->digit_size == 4) { + n = mpd_qexport_u32((uint32_t **)&tmp_digits, 0, base, x, &status); + } + else { + n = mpd_qexport_u16((uint16_t **)&tmp_digits, 0, base, x, &status); + } if (n == SIZE_MAX) { PyErr_NoMemory(); mpd_del(x); + mpd_free(tmp_digits); return NULL; } - if (n == 1) { - sdigit val = mpd_arith_sign(x) * ob_digit[0]; - mpd_free(ob_digit); - mpd_del(x); - return PyLong_FromLong(val); - } + void *digits; + PyLongWriter *writer = PyLongWriter_Create(mpd_isnegative(x), n, &digits); - assert(n > 0); - assert(!mpd_iszero(x)); - pylong = _PyLong_FromDigits(mpd_isnegative(x), n, ob_digit); - mpd_free(ob_digit); mpd_del(x); - return (PyObject *) pylong; + if (writer == NULL) { + mpd_free(tmp_digits); + return NULL; + } + memcpy(digits, tmp_digits, layout->digit_size*n); + mpd_free(tmp_digits); + return PyLongWriter_Finish(writer); } /* Convert a Decimal to its exact integer ratio representation. */

1 0

gh-119182: Use public PyUnicodeWriter in _json.c (#129249)
by vstinner Jan. 24, 2025

Jan. 24, 2025

https://github.com/python/cpython/commit/6e63c4736beebdf912acd391fc437672ee… commit: 6e63c4736beebdf912acd391fc437672ee9d362e branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-24T11:38:13+01:00 summary: gh-119182: Use public PyUnicodeWriter in _json.c (#129249) Replace the private _PyUnicodeWriter API with the public PyUnicodeWriter API. files: M Modules/_json.c diff --git a/Modules/_json.c b/Modules/_json.c index a99abbe72bf7a0..091bcbfdced42b 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -353,6 +353,13 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { return tpl; } +static inline int +_PyUnicodeWriter_IsEmpty(PyUnicodeWriter *writer_pub) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)writer_pub; + return (writer->pos == 0); +} + static PyObject * scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) { @@ -371,9 +378,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next const void *buf; int kind; - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto bail; + } len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); @@ -404,7 +412,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next if (c == '"') { // Fast path for simple case. - if (writer.buffer == NULL) { + if (_PyUnicodeWriter_IsEmpty(writer)) { PyObject *ret = PyUnicode_Substring(pystr, end, next); if (ret == NULL) { goto bail; @@ -420,7 +428,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next /* Pick up this chunk if it's not zero length */ if (next != end) { - if (_PyUnicodeWriter_WriteSubstring(&writer, pystr, end, next) < 0) { + if (PyUnicodeWriter_WriteSubstring(writer, pystr, end, next) < 0) { goto bail; } } @@ -511,18 +519,18 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next end -= 6; } } - if (_PyUnicodeWriter_WriteChar(&writer, c) < 0) { + if (PyUnicodeWriter_WriteChar(writer, c) < 0) { goto bail; } } - rval = _PyUnicodeWriter_Finish(&writer); + rval = PyUnicodeWriter_Finish(writer); *next_end_ptr = end; return rval; bail: *next_end_ptr = -1; - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; }

1 0

gh-129185: Use PyMutex in tracemalloc (#129246)
by vstinner Jan. 24, 2025

Jan. 24, 2025

https://github.com/python/cpython/commit/c005ea4951581bd5da7594f29594c9c840… commit: c005ea4951581bd5da7594f29594c9c840ec2fa5 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-24T11:25:24+01:00 summary: gh-129185: Use PyMutex in tracemalloc (#129246) files: M Include/internal/pycore_tracemalloc.h M Python/tracemalloc.c diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 3fb0aa133fc41d..572e8025876319 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -70,7 +70,7 @@ struct _tracemalloc_runtime_state { PyMemAllocatorEx obj; } allocators; - PyThread_type_lock tables_lock; + PyMutex tables_lock; /* Size in bytes of currently traced memory. Protected by TABLES_LOCK(). */ size_t traced_memory; diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 20651016a80ec2..d27c2f9319ae58 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -3,6 +3,7 @@ #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() @@ -37,8 +38,8 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ #define tables_lock _PyRuntime.tracemalloc.tables_lock -#define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) -#define TABLES_UNLOCK() PyThread_release_lock(tables_lock) +#define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) +#define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) #define DEFAULT_DOMAIN 0 @@ -741,13 +742,6 @@ _PyTraceMalloc_Init(void) return _PyStatus_NO_MEMORY(); } - if (tables_lock == NULL) { - tables_lock = PyThread_allocate_lock(); - if (tables_lock == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - tracemalloc_filenames = hashtable_new(hashtable_hash_pyobject, hashtable_compare_unicode, tracemalloc_clear_filename, NULL); @@ -792,11 +786,6 @@ tracemalloc_deinit(void) _Py_hashtable_destroy(tracemalloc_tracebacks); _Py_hashtable_destroy(tracemalloc_filenames); - if (tables_lock != NULL) { - PyThread_free_lock(tables_lock); - tables_lock = NULL; - } - PyThread_tss_delete(&tracemalloc_reentrant_key); }

1 0

gh-129173: Use `_PyUnicodeError_GetParams` in `PyCodec_IgnoreErrors` (#129174)
by picnixz Jan. 24, 2025

Jan. 24, 2025

https://github.com/python/cpython/commit/36bb22993307b25593237a1022790d329b… commit: 36bb22993307b25593237a1022790d329b7c75e0 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-24T11:25:03+01:00 summary: gh-129173: Use `_PyUnicodeError_GetParams` in `PyCodec_IgnoreErrors` (#129174) We also cleanup `PyCodec_StrictErrors` and the error message rendered when an object of incorrect type is passed to codec error handlers. files: M Python/codecs.c diff --git a/Python/codecs.c b/Python/codecs.c index 07eaa8ecddcae0..53680a79082634 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -659,44 +659,64 @@ PyObject *PyCodec_LookupError(const char *name) return handler; } -static void wrong_exception_type(PyObject *exc) + +static inline void +wrong_exception_type(PyObject *exc) { PyErr_Format(PyExc_TypeError, - "don't know how to handle %.200s in error callback", - Py_TYPE(exc)->tp_name); + "don't know how to handle %T in error callback", exc); } + +#define _PyIsUnicodeEncodeError(EXC) \ + PyObject_TypeCheck(EXC, (PyTypeObject *)PyExc_UnicodeEncodeError) +#define _PyIsUnicodeDecodeError(EXC) \ + PyObject_TypeCheck(EXC, (PyTypeObject *)PyExc_UnicodeDecodeError) +#define _PyIsUnicodeTranslateError(EXC) \ + PyObject_TypeCheck(EXC, (PyTypeObject *)PyExc_UnicodeTranslateError) + + +// --- handler: 'strict' ------------------------------------------------------ + PyObject *PyCodec_StrictErrors(PyObject *exc) { - if (PyExceptionInstance_Check(exc)) + if (PyExceptionInstance_Check(exc)) { PyErr_SetObject(PyExceptionInstance_Class(exc), exc); - else + } + else { PyErr_SetString(PyExc_TypeError, "codec must pass exception instance"); + } return NULL; } -PyObject *PyCodec_IgnoreErrors(PyObject *exc) +// --- handler: 'ignore' ------------------------------------------------------ + +static PyObject * +_PyCodec_IgnoreError(PyObject *exc, int as_bytes) { Py_ssize_t end; - - if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; + if (_PyUnicodeError_GetParams(exc, NULL, NULL, NULL, + &end, NULL, as_bytes) < 0) + { + return NULL; } - else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) { - if (PyUnicodeDecodeError_GetEnd(exc, &end)) - return NULL; + return Py_BuildValue("(Nn)", Py_GetConstant(Py_CONSTANT_EMPTY_STR), end); +} + + +PyObject *PyCodec_IgnoreErrors(PyObject *exc) +{ + if (_PyIsUnicodeEncodeError(exc) || _PyIsUnicodeTranslateError(exc)) { + return _PyCodec_IgnoreError(exc, false); } - else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) { - if (PyUnicodeTranslateError_GetEnd(exc, &end)) - return NULL; + else if (_PyIsUnicodeDecodeError(exc)) { + return _PyCodec_IgnoreError(exc, true); } else { wrong_exception_type(exc); return NULL; } - return Py_BuildValue("(Nn)", Py_GetConstant(Py_CONSTANT_EMPTY_STR), end); } @@ -1368,13 +1388,17 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) } -static PyObject *strict_errors(PyObject *self, PyObject *exc) +// --- Codecs registry handlers ----------------------------------------------- + +static inline PyObject * +strict_errors(PyObject *Py_UNUSED(self), PyObject *exc) { return PyCodec_StrictErrors(exc); } -static PyObject *ignore_errors(PyObject *self, PyObject *exc) +static inline PyObject * +ignore_errors(PyObject *Py_UNUSED(self), PyObject *exc) { return PyCodec_IgnoreErrors(exc); }

1 0

gh-119182: Use public PyUnicodeWriter in stringio.c (#129243)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/732670d93b9b0c0ff8adde07418fd6f839… commit: 732670d93b9b0c0ff8adde07418fd6f8397893ef branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T23:31:49Z summary: gh-119182: Use public PyUnicodeWriter in stringio.c (#129243) Replace the private _PyUnicodeWriter API with the public PyUnicodeWriter API. files: M Modules/_io/stringio.c diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 687f584ee6caff..d1a71298a9087f 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -30,7 +30,7 @@ typedef struct { _PyUnicodeWriter is destroyed. */ int state; - _PyUnicodeWriter writer; + PyUnicodeWriter *writer; char ok; /* initialized? */ char closed; @@ -129,14 +129,18 @@ resize_buffer(stringio *self, size_t size) static PyObject * make_intermediate(stringio *self) { - PyObject *intermediate = _PyUnicodeWriter_Finish(&self->writer); + PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; self->state = STATE_REALIZED; if (intermediate == NULL) return NULL; - _PyUnicodeWriter_Init(&self->writer); - self->writer.overallocate = 1; - if (_PyUnicodeWriter_WriteStr(&self->writer, intermediate)) { + self->writer = PyUnicodeWriter_Create(0); + if (self->writer == NULL) { + Py_DECREF(intermediate); + return NULL; + } + if (PyUnicodeWriter_WriteStr(self->writer, intermediate)) { Py_DECREF(intermediate); return NULL; } @@ -155,7 +159,8 @@ realize(stringio *self) assert(self->state == STATE_ACCUMULATING); self->state = STATE_REALIZED; - intermediate = _PyUnicodeWriter_Finish(&self->writer); + intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; if (intermediate == NULL) return -1; @@ -217,7 +222,7 @@ write_str(stringio *self, PyObject *obj) if (self->state == STATE_ACCUMULATING) { if (self->string_size == self->pos) { - if (_PyUnicodeWriter_WriteStr(&self->writer, decoded)) + if (PyUnicodeWriter_WriteStr(self->writer, decoded)) goto fail; goto success; } @@ -577,7 +582,8 @@ _io_StringIO_close_impl(stringio *self) /* Free up some memory */ if (resize_buffer(self, 0) < 0) return NULL; - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); + self->writer = NULL; Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); @@ -615,7 +621,7 @@ stringio_dealloc(stringio *self) PyMem_Free(self->buf); self->buf = NULL; } - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(self); if (self->weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); @@ -699,7 +705,8 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->ok = 0; - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); + self->writer = NULL; Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); @@ -754,8 +761,10 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, /* Empty stringio object, we can start by accumulating */ if (resize_buffer(self, 0) < 0) return -1; - _PyUnicodeWriter_Init(&self->writer); - self->writer.overallocate = 1; + self->writer = PyUnicodeWriter_Create(0); + if (self->writer == NULL) { + return -1; + } self->state = STATE_ACCUMULATING; } self->pos = 0;

1 0

gh-109975: Add list of 3.13 removed library replacements (#127816)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/bab8918f9a647c20b64f5c165b45c0926f… commit: bab8918f9a647c20b64f5c165b45c0926f19ca0d branch: main author: Jeong, YunWon <69878+youknowone(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T22:20:40+01:00 summary: gh-109975: Add list of 3.13 removed library replacements (#127816) Add list of 3.13 removed library replacements files: M Doc/whatsnew/3.13.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c2cfdf3186f86d..f2a8c711f4df88 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1499,8 +1499,20 @@ All of the following modules were deprecated in Python 3.11, and are now removed: * :mod:`!aifc` + + * :pypi:`standard-aifc`: + Use the redistribution of ``aifc`` library from PyPI. + * :mod:`!audioop` + + * :pypi:`audioop-lts`: + Use ``audioop-lts`` library from PyPI. + * :mod:`!chunk` + + * :pypi:`standard-chunk`: + Use the redistribution of ``chunk`` library from PyPI. + * :mod:`!cgi` and :mod:`!cgitb` * :class:`!cgi.FieldStorage` can typically be replaced with @@ -1531,6 +1543,9 @@ and are now removed: For example, the :class:`email.message.EmailMessage` and :class:`email.message.Message` classes. + * :pypi:`standard-cgi`: and :pypi:`standard-cgitb`: + Use the redistribution of ``cgi`` and ``cgitb`` library from PyPI. + * :mod:`!crypt` and the private :mod:`!_crypt` extension. The :mod:`hashlib` module may be an appropriate replacement when simply hashing a value is required. @@ -1549,6 +1564,8 @@ and are now removed: Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library call and associated functionality. + * :pypi:`standard-crypt` and :pypi:`deprecated-crypt-alternative`: + Use the redistribution of ``crypt`` and reimplementation of ``_crypt`` libraries from PyPI. * :mod:`!imghdr`: The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries @@ -1556,30 +1573,65 @@ and are now removed: For example, the :func:`!puremagic.what` function can be used to replace the :func:`!imghdr.what` function for all file formats that were supported by :mod:`!imghdr`. + + * :pypi:`standard-imghdr`: + Use the redistribution of ``imghdr`` library from PyPI. + * :mod:`!mailcap`: Use the :mod:`mimetypes` module instead. + + * :pypi:`standard-mailcap`: + Use the redistribution of ``mailcap`` library from PyPI. + * :mod:`!msilib` * :mod:`!nis` * :mod:`!nntplib`: Use the :pypi:`pynntp` library from PyPI instead. + + * :pypi:`standard-nntplib`: + Use the redistribution of ``nntplib`` library from PyPI. + * :mod:`!ossaudiodev`: For audio playback, use the :pypi:`pygame` library from PyPI instead. * :mod:`!pipes`: Use the :mod:`subprocess` module instead. Use :func:`shlex.quote` to replace the undocumented ``pipes.quote`` function. + + * :pypi:`standard-pipes`: + Use the redistribution of ``pipes`` library from PyPI. + * :mod:`!sndhdr`: The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries should be used as replacements. + + * :pypi:`standard-sndhdr`: + Use the redistribution of ``sndhdr`` library from PyPI. + * :mod:`!spwd`: Use the :pypi:`python-pam` library from PyPI instead. * :mod:`!sunau` + + * :pypi:`standard-sunau`: + Use the redistribution of ``sunau`` library from PyPI. + * :mod:`!telnetlib`, Use the :pypi:`telnetlib3` or :pypi:`Exscript` libraries from PyPI instead. + + * :pypi:`standard-telnetlib`: + Use the redistribution of ``telnetlib`` library from PyPI. + * :mod:`!uu`: Use the :mod:`base64` module instead, as a modern alternative. + + * :pypi:`standard-uu`: + Use the redistribution of ``uu`` library from PyPI. + * :mod:`!xdrlib` + * :pypi:`standard-xdrlib`: + Use the redistribution of ``xdrlib`` library from PyPI. + (Contributed by Victor Stinner and Zachary Ware in :gh:`104773` and :gh:`104780`.)

1 0

[3.13] gh-97850: Remove the mention of removal from `ResourceReader` docs (GH-128602) (#128654)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/eed7865ceea83f56e46307c9dc78cb5352… commit: eed7865ceea83f56e46307c9dc78cb53526071f6 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T22:17:32+01:00 summary: [3.13] gh-97850: Remove the mention of removal from `ResourceReader` docs (GH-128602) (#128654) gh-97850: Remove the mention of removal from `ResourceReader` docs (GH-128602) Remove the mention of removal from ResourceReader docs (cherry picked from commit a1284e97979ff73ad72ad06c796b904137950576) Co-authored-by: Tomas R <tomas.roun8(a)gmail.com> files: M Doc/library/importlib.resources.abc.rst diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 54995ddbfbca12..4085bdf6598d98 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -43,7 +43,7 @@ :const:`None`. An object compatible with this ABC should only be returned when the specified module is a package. - .. deprecated-removed:: 3.12 3.14 + .. deprecated:: 3.12 Use :class:`importlib.resources.abc.TraversableResources` instead. .. abstractmethod:: open_resource(resource)

1 0

[3.13] gh-126172: Fix a misleading statement in PYTHON_BASIC_REPL documentation (GH-127203) (#129152)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/6a261a02a72bfa1b8e3e1c8fb70b339be6… commit: 6a261a02a72bfa1b8e3e1c8fb70b339be6033df2 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T22:17:11+01:00 summary: [3.13] gh-126172: Fix a misleading statement in PYTHON_BASIC_REPL documentation (GH-127203) (#129152) gh-126172: Fix a misleading statement in PYTHON_BASIC_REPL documentation (GH-127203) Fix a misleading statement in PYTHON_BASIC_REPL documentation. (cherry picked from commit f18b2264929c56360c868d7ad77508035d751352) Co-authored-by: Michael Kashirin <88917554+mkashirin(a)users.noreply.github.com> files: M Doc/using/cmdline.rst diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 8f54a0fdebe1d3..797ba29ad5476c 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1197,7 +1197,7 @@ conflict. .. envvar:: PYTHON_BASIC_REPL - If this variable is set to ``1``, the interpreter will not attempt to + If this variable is set to any value, the interpreter will not attempt to load the Python-based :term:`REPL` that requires :mod:`curses` and :mod:`readline`, and will instead use the traditional parser-based :term:`REPL`.

1 0

[3.12] gh-128479: fix asyncio staggered race leaking tasks, and logging unhandled exception.append exception (GH-128475) (#129228)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/e94939cedb76650c06821e91d9011c0fe8… commit: e94939cedb76650c06821e91d9011c0fe82fd1ec branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T22:16:21+01:00 summary: [3.12] gh-128479: fix asyncio staggered race leaking tasks, and logging unhandled exception.append exception (GH-128475) (#129228) gh-128479: fix asyncio staggered race leaking tasks, and logging unhandled exception.append exception (GH-128475) (cherry picked from commit ec91e1c2762412f1408b0dfb5d281873b852affe) Co-authored-by: Thomas Grainger <tagrain(a)gmail.com> Co-authored-by: Peter Bierma <zintensitydev(a)gmail.com> files: A Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst M Lib/asyncio/staggered.py M Lib/test/test_asyncio/test_staggered.py diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 0f4df8855a80b9..0afed64fdf9c0f 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -66,8 +66,27 @@ async def staggered_race(coro_fns, delay, *, loop=None): enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None + unhandled_exceptions = [] exceptions = [] - running_tasks = [] + running_tasks = set() + on_completed_fut = None + + def task_done(task): + running_tasks.discard(task) + if ( + on_completed_fut is not None + and not on_completed_fut.done() + and not running_tasks + ): + on_completed_fut.set_result(None) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + unhandled_exceptions.append(exc) async def run_one_coro(ok_to_start, previous_failed) -> None: # in eager tasks this waits for the calling task to append this task @@ -91,11 +110,11 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: this_failed = locks.Event() next_ok_to_start = locks.Event() next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) - running_tasks.append(next_task) + running_tasks.add(next_task) + next_task.add_done_callback(task_done) # next_task has been appended to running_tasks so next_task is ok to # start. next_ok_to_start.set() - assert len(running_tasks) == this_index + 2 # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -120,31 +139,36 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: # up as done() == True, cancelled() == False, exception() == # asyncio.CancelledError. This behavior is specified in # https://bugs.python.org/issue30048 - for i, t in enumerate(running_tasks): - if i != this_index: + current_task = tasks.current_task(loop) + for t in running_tasks: + if t is not current_task: t.cancel() - ok_to_start = locks.Event() - first_task = loop.create_task(run_one_coro(ok_to_start, None)) - running_tasks.append(first_task) - # first_task has been appended to running_tasks so first_task is ok to start. - ok_to_start.set() + propagate_cancellation_error = None try: - # Wait for a growing list of tasks to all finish: poor man's version of - # curio's TaskGroup or trio's nursery - done_count = 0 - while done_count != len(running_tasks): - done, _ = await tasks.wait(running_tasks) - done_count = len(done) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) + running_tasks.add(first_task) + first_task.add_done_callback(task_done) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() + propagate_cancellation_error = None + # Make sure no tasks are left running if we leave this function + while running_tasks: + on_completed_fut = loop.create_future() + try: + await on_completed_fut + except exceptions_mod.CancelledError as ex: + propagate_cancellation_error = ex + for task in running_tasks: + task.cancel(*ex.args) + on_completed_fut = None + if __debug__ and unhandled_exceptions: # If run_one_coro raises an unhandled exception, it's probably a # programming error, and I want to see it. - if __debug__: - for d in done: - if d.done() and not d.cancelled() and d.exception(): - raise d.exception() + raise ExceptionGroup("staggered race failed", unhandled_exceptions) + if propagate_cancellation_error is not None: + raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - del exceptions - # Make sure no tasks are left running if we leave this function - for t in running_tasks: - t.cancel() + del exceptions, propagate_cancellation_error, unhandled_exceptions diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 74941f704c4890..40455a3804e3dd 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -122,3 +122,30 @@ async def do_set(): self.assertIsNone(excs[0], None) self.assertIsInstance(excs[1], asyncio.CancelledError) self.assertIsInstance(excs[2], asyncio.CancelledError) + + + async def test_cancelled(self): + log = [] + with self.assertRaises(TimeoutError): + async with asyncio.timeout(None) as cs_outer, asyncio.timeout(None) as cs_inner: + async def coro_fn(): + cs_inner.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 1") + + cs_outer.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 2") + try: + await staggered_race([coro_fn], delay=None) + except asyncio.CancelledError: + log.append("cancelled 3") + raise + + self.assertListEqual(log, ["cancelled 1", "cancelled 2", "cancelled 3"]) diff --git a/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst new file mode 100644 index 00000000000000..fc3b4d5a5273a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst @@ -0,0 +1 @@ +Fix :func:`!asyncio.staggered.staggered_race` leaking tasks and issuing an unhandled exception.

1 0