Mailman 3 January 2025 - Python-checkins

gh-129149: Add fast path in PYLONG_FROM_UINT macro for compact integers (#129168)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/ab353b31bb300d55e845476a8e26536645… commit: ab353b31bb300d55e845476a8e265366455d93fc branch: main author: Srinivas Reddy Thatiparthy (తాటిపర్తి శ్రీనివాస్ రెడ్డి) <thatiparthysreenivas(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T14:36:52Z summary: gh-129149: Add fast path in PYLONG_FROM_UINT macro for compact integers (#129168) Add fast path in PyLong_From*() functions for compact integers. Co-authored-by: Pieter Eendebak <pieter.eendebak(a)gmail.com> Co-authored-by: Sergey B Kirpichev <skirpichev(a)gmail.com> Co-authored-by: Yan Yanchii <yyanchiy(a)gmail.com> Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst M Objects/longobject.c diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst new file mode 100644 index 00000000000000..d946f6a2fea185 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst @@ -0,0 +1,2 @@ +Add fast path for medium-size integers in :c:func:`PyLong_FromUnsignedLong`, +:c:func:`PyLong_FromUnsignedLongLong` and :c:func:`PyLong_FromSize_t`. diff --git a/Objects/longobject.c b/Objects/longobject.c index 89526d5a430304..b4e3a70adf2b5b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -365,9 +365,13 @@ PyLong_FromLong(long ival) if (IS_SMALL_UINT(ival)) { \ return get_small_int((sdigit)(ival)); \ } \ + if ((ival) <= PyLong_MASK) { \ + return _PyLong_FromMedium((sdigit)(ival)); \ + } \ + /* Do shift in two steps to avoid possible undefined behavior. */ \ + INT_TYPE t = (ival) >> PyLong_SHIFT >> PyLong_SHIFT; \ /* Count the number of Python digits. */ \ - Py_ssize_t ndigits = 0; \ - INT_TYPE t = (ival); \ + Py_ssize_t ndigits = 2; \ while (t) { \ ++ndigits; \ t >>= PyLong_SHIFT; \

1 0

gh-101410: support custom messages for domain errors in the math module (#124299)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/75f59bb62938fc9acac7a4ede19404e889… commit: 75f59bb62938fc9acac7a4ede19404e889417c6b branch: main author: Sergey B Kirpichev <skirpichev(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T13:55:25Z summary: gh-101410: support custom messages for domain errors in the math module (#124299) This adds basic support to override default messages for domain errors in the math_1() helper. The sqrt(), atanh(), log2(), log10() and log() functions were modified as examples. New macro supports gradual changing of error messages in other 1-arg functions. Co-authored-by: CharlieZhao <zhaoyu_hit(a)qq.com> Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst M Lib/test/test_math.py M Modules/mathmodule.c diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 6976a5d85da019..2c57d288bc03ff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2503,6 +2503,46 @@ def test_input_exceptions(self): self.assertRaises(TypeError, math.atan2, 1.0) self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + def test_exception_messages(self): + x = -1.1 + with self.assertRaisesRegex(ValueError, + f"expected a nonnegative input, got {x}"): + math.sqrt(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(123, x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x, 123) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log2(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log10(x) + x = decimal.Decimal('-1.1') + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + x = fractions.Fraction(1, 10**400) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {float(x)}"): + math.log(x) + x = -123 + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a float or nonnegative integer, got {x}"): + math.gamma(x) + x = 1.0 + with self.assertRaisesRegex(ValueError, + f"expected a number between -1 and 1, got {x}"): + math.atanh(x) + # Custom assertions. def assertIsNaN(self, value): diff --git a/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst b/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst new file mode 100644 index 00000000000000..31493686daec97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst @@ -0,0 +1,3 @@ +Support custom messages for domain errors in the :mod:`math` module +(:func:`math.sqrt`, :func:`math.log` and :func:`math.atanh` were modified as +examples). Patch by Charlie Zhao and Sergey B Kirpichev. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 29638114dd94a9..b4c15a143f9838 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -858,12 +858,15 @@ math_lcm_impl(PyObject *module, PyObject * const *args, * true (1), but may return false (0) without setting up an exception. */ static int -is_error(double x) +is_error(double x, int raise_edom) { int result = 1; /* presumption of guilt */ assert(errno); /* non-zero errno is a precondition for calling */ - if (errno == EDOM) - PyErr_SetString(PyExc_ValueError, "math domain error"); + if (errno == EDOM) { + if (raise_edom) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + } else if (errno == ERANGE) { /* ANSI C generally requires libm functions to set ERANGE @@ -928,7 +931,8 @@ is_error(double x) */ static PyObject * -math_1(PyObject *arg, double (*func) (double), int can_overflow) +math_1(PyObject *arg, double (*func) (double), int can_overflow, + const char *err_msg) { double x, r; x = PyFloat_AsDouble(arg); @@ -936,25 +940,34 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) return NULL; errno = 0; r = (*func)(x); - if (isnan(r) && !isnan(x)) { - PyErr_SetString(PyExc_ValueError, - "math domain error"); /* invalid arg */ - return NULL; - } + if (isnan(r) && !isnan(x)) + goto domain_err; /* domain error */ if (isinf(r) && isfinite(x)) { if (can_overflow) PyErr_SetString(PyExc_OverflowError, "math range error"); /* overflow */ else - PyErr_SetString(PyExc_ValueError, - "math domain error"); /* singularity */ + goto domain_err; /* singularity */ return NULL; } - if (isfinite(r) && errno && is_error(r)) + if (isfinite(r) && errno && is_error(r, 1)) /* this branch unnecessary on most platforms */ return NULL; return PyFloat_FromDouble(r); + +domain_err: + if (err_msg) { + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, err_msg, buf); + PyMem_Free(buf); + } + } + else { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + return NULL; } /* variant of math_1, to be used when the function being wrapped is known to @@ -962,7 +975,7 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) errno = ERANGE for overflow). */ static PyObject * -math_1a(PyObject *arg, double (*func) (double)) +math_1a(PyObject *arg, double (*func) (double), const char *err_msg) { double x, r; x = PyFloat_AsDouble(arg); @@ -970,8 +983,17 @@ math_1a(PyObject *arg, double (*func) (double)) return NULL; errno = 0; r = (*func)(x); - if (errno && is_error(r)) + if (errno && is_error(r, err_msg ? 0 : 1)) { + if (err_msg && errno == EDOM) { + assert(!PyErr_Occurred()); /* exception is not set by is_error() */ + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, err_msg, buf); + PyMem_Free(buf); + } + } return NULL; + } return PyFloat_FromDouble(r); } @@ -1031,7 +1053,7 @@ math_2(PyObject *const *args, Py_ssize_t nargs, else errno = 0; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r); @@ -1039,13 +1061,25 @@ math_2(PyObject *const *args, Py_ssize_t nargs, #define FUNC1(funcname, func, can_overflow, docstring) \ static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ - return math_1(args, func, can_overflow); \ + return math_1(args, func, can_overflow, NULL); \ + }\ + PyDoc_STRVAR(math_##funcname##_doc, docstring); + +#define FUNC1D(funcname, func, can_overflow, docstring, err_msg) \ + static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ + return math_1(args, func, can_overflow, err_msg); \ }\ PyDoc_STRVAR(math_##funcname##_doc, docstring); #define FUNC1A(funcname, func, docstring) \ static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ - return math_1a(args, func); \ + return math_1a(args, func, NULL); \ + }\ + PyDoc_STRVAR(math_##funcname##_doc, docstring); + +#define FUNC1AD(funcname, func, docstring, err_msg) \ + static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ + return math_1a(args, func, err_msg); \ }\ PyDoc_STRVAR(math_##funcname##_doc, docstring); @@ -1077,9 +1111,10 @@ FUNC2(atan2, atan2, "atan2($module, y, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of y/x.\n\n" "Unlike atan(y/x), the signs of both x and y are considered.") -FUNC1(atanh, atanh, 0, +FUNC1D(atanh, atanh, 0, "atanh($module, x, /)\n--\n\n" - "Return the inverse hyperbolic tangent of x.") + "Return the inverse hyperbolic tangent of x.", + "expected a number between -1 and 1, got %s") FUNC1(cbrt, cbrt, 0, "cbrt($module, x, /)\n--\n\n" "Return the cube root of x.") @@ -1190,9 +1225,10 @@ math_floor(PyObject *module, PyObject *number) return PyLong_FromDouble(floor(x)); } -FUNC1A(gamma, m_tgamma, +FUNC1AD(gamma, m_tgamma, "gamma($module, x, /)\n--\n\n" - "Gamma function at x.") + "Gamma function at x.", + "expected a float or nonnegative integer, got %s") FUNC1A(lgamma, m_lgamma, "lgamma($module, x, /)\n--\n\n" "Natural logarithm of absolute value of Gamma function at x.") @@ -1212,9 +1248,10 @@ FUNC1(sin, sin, 0, FUNC1(sinh, sinh, 1, "sinh($module, x, /)\n--\n\n" "Return the hyperbolic sine of x.") -FUNC1(sqrt, sqrt, 0, +FUNC1D(sqrt, sqrt, 0, "sqrt($module, x, /)\n--\n\n" - "Return the square root of x.") + "Return the square root of x.", + "expected a nonnegative input, got %s") FUNC1(tan, tan, 0, "tan($module, x, /)\n--\n\n" "Return the tangent of x (measured in radians).") @@ -2141,7 +2178,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = ERANGE; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; return PyFloat_FromDouble(r); } @@ -2195,8 +2232,8 @@ loghelper(PyObject* arg, double (*func)(double)) /* Negative or zero inputs give a ValueError. */ if (!_PyLong_IsPositive((PyLongObject *)arg)) { - PyErr_SetString(PyExc_ValueError, - "math domain error"); + PyErr_Format(PyExc_ValueError, + "expected a positive input, got %S", arg); return NULL; } @@ -2220,7 +2257,7 @@ loghelper(PyObject* arg, double (*func)(double)) } /* Else let libm handle it by itself. */ - return math_1(arg, func, 0); + return math_1(arg, func, 0, "expected a positive input, got %s"); } @@ -2369,7 +2406,7 @@ math_fmod_impl(PyObject *module, double x, double y) else errno = 0; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r); @@ -3010,7 +3047,7 @@ math_pow_impl(PyObject *module, double x, double y) } } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r);

1 0

[3.12] gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) (#129217) (#129221)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/bb7c54d5ba6492b01b1c8731207a36196f… commit: bb7c54d5ba6492b01b1c8731207a36196f85ea8b branch: 3.12 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T13:29:46Z summary: [3.12] gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) (#129217) (#129221) [3.13] gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) (#129217) gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) Support calling PyTraceMalloc_Track() and PyTraceMalloc_Untrack() during late Python finalization. * Call _PyTraceMalloc_Fini() later in Python finalization. * Test also PyTraceMalloc_Untrack() without the GIL * PyTraceMalloc_Untrack() now gets the GIL. * Test also PyTraceMalloc_Untrack() in test_tracemalloc_track_race(). (cherry picked from commit 46c7e13c055c218e18b0424efc60965e6a5fe6ea) (cherry picked from commit e3b3e01d6a6f43c15890d14f139f38155601643a) files: M Lib/test/test_tracemalloc.py M Modules/_testcapi/mem.c M Modules/_testcapimodule.c M Python/pylifecycle.c M Python/tracemalloc.c diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 97103a32e02ed0..316216554abebe 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1,6 +1,7 @@ import contextlib import os import sys +import textwrap import tracemalloc import unittest from unittest.mock import patch @@ -16,6 +17,7 @@ _testcapi = None +DEFAULT_DOMAIN = 0 EMPTY_STRING_SIZE = sys.getsizeof(b'') INVALID_NFRAME = (-1, 2**30) @@ -1020,8 +1022,8 @@ def track(self, release_gil=False, nframe=1): release_gil) return frames - def untrack(self): - _testcapi.tracemalloc_untrack(self.domain, self.ptr) + def untrack(self, release_gil=False): + _testcapi.tracemalloc_untrack(self.domain, self.ptr, release_gil) def get_traced_memory(self): # Get the traced size in the domain @@ -1063,7 +1065,7 @@ def test_track_already_tracked(self): self.assertEqual(self.get_traceback(), tracemalloc.Traceback(frames)) - def test_untrack(self): + def check_untrack(self, release_gil): tracemalloc.start() self.track() @@ -1071,13 +1073,19 @@ def test_untrack(self): self.assertEqual(self.get_traced_memory(), self.size) # untrack must remove the trace - self.untrack() + self.untrack(release_gil) self.assertIsNone(self.get_traceback()) self.assertEqual(self.get_traced_memory(), 0) # calling _PyTraceMalloc_Untrack() multiple times must not crash - self.untrack() - self.untrack() + self.untrack(release_gil) + self.untrack(release_gil) + + def test_untrack(self): + self.check_untrack(False) + + def test_untrack_without_gil(self): + self.check_untrack(True) def test_stop_track(self): tracemalloc.start() @@ -1103,6 +1111,29 @@ def test_tracemalloc_track_race(self): # gh-128679: Test fix for tracemalloc.stop() race condition _testcapi.tracemalloc_track_race() + def test_late_untrack(self): + code = textwrap.dedent(f""" + from test import support + import tracemalloc + import _testcapi + + class Tracked: + def __init__(self, domain, size): + self.domain = domain + self.ptr = id(self) + self.size = size + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size) + + def __del__(self, untrack=_testcapi.tracemalloc_untrack): + untrack(self.domain, self.ptr, 1) + + domain = {DEFAULT_DOMAIN} + tracemalloc.start() + obj = Tracked(domain, 1024 * 1024) + support.late_deletion(obj) + """) + assert_python_ok("-c", code) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 545cc6b669c010..b40cacb0a459b3 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -637,8 +637,9 @@ tracemalloc_untrack(PyObject *self, PyObject *args) { unsigned int domain; PyObject *ptr_obj; + int release_gil = 0; - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + if (!PyArg_ParseTuple(args, "IO|i", &domain, &ptr_obj, &release_gil)) { return NULL; } void *ptr = PyLong_AsVoidPtr(ptr_obj); @@ -646,7 +647,15 @@ tracemalloc_untrack(PyObject *self, PyObject *args) return NULL; } - int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + } if (res < 0) { PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); return NULL; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1fbeb58e3d404b..36b8fcbcddf479 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3231,6 +3231,7 @@ static void tracemalloc_track_race_thread(void *data) { PyTraceMalloc_Track(123, 10, 1); + PyTraceMalloc_Untrack(123, 10); PyThread_type_lock lock = (PyThread_type_lock)data; PyThread_release_lock(lock); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f301026fa5befd..ef4e60765705b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1933,7 +1933,7 @@ Py_FinalizeEx(void) /* Disable tracemalloc after all Python objects have been destroyed, so it is possible to use tracemalloc in objects destructor. */ - _PyTraceMalloc_Fini(); + _PyTraceMalloc_Stop(); /* Finalize any remaining import state */ // XXX Move these up to where finalize_modules() is currently. @@ -1986,6 +1986,8 @@ Py_FinalizeEx(void) finalize_interp_clear(tstate); + _PyTraceMalloc_Fini(); + #ifdef WITH_PYMALLOC if (malloc_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 9e653770c3eab5..852e5b03917af6 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1313,29 +1313,48 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini(). + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int res; if (tracemalloc_config.tracing) { - res = tracemalloc_add_trace(domain, ptr, size); + result = tracemalloc_add_trace(domain, ptr, size); } else { // gh-128679: tracemalloc.stop() was called by another thread - res = -2; + result = -2; } TABLES_UNLOCK(); +done: PyGILState_Release(gil_state); - return res; + return result; } int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { + // Need the GIL to prevent races on the first 'tracing' test + PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini() + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int result; if (tracemalloc_config.tracing) { tracemalloc_remove_trace(domain, ptr); result = 0; @@ -1346,6 +1365,8 @@ PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) } TABLES_UNLOCK(); +done: + PyGILState_Release(gil_state); return result; }

1 0

gh-126004: Fix positions handling in `codecs.backslashreplace_errors` (#127676)
by picnixz Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/25a614a502e607a333f7aa7f4d69e6f2a4… commit: 25a614a502e607a333f7aa7f4d69e6f2a47148ad branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-23T14:28:33+01:00 summary: gh-126004: Fix positions handling in `codecs.backslashreplace_errors` (#127676) This fixes how `PyCodec_BackslashReplaceErrors` handles the `start` and `end` attributes of `UnicodeError` objects via the `_PyUnicodeError_GetParams` helper. files: A Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst M Lib/test/test_capi/test_codecs.py M Python/codecs.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index f57191ddcdbeb4..e368f70c196abf 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -849,7 +849,8 @@ def test_codec_xmlcharrefreplace_errors_handler(self): def test_codec_backslashreplace_errors_handler(self): handler = _testcapi.codec_backslashreplace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors, + safe=True) def test_codec_namereplace_errors_handler(self): handler = _testlimitedcapi.codec_namereplace_errors diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst new file mode 100644 index 00000000000000..619d73042a9bb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.backslashreplace_errors` error handler. Patch by +Bénédikt Tran. diff --git a/Python/codecs.c b/Python/codecs.c index b657dd134a668e..07eaa8ecddcae0 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -864,108 +864,107 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) { - PyObject *object; - Py_ssize_t i; - Py_ssize_t start; - Py_ssize_t end; - PyObject *res; - Py_UCS1 *outp; - int ressize; - Py_UCS4 c; - + PyObject *obj; + Py_ssize_t objlen, start, end, slen; if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) { - const unsigned char *p; - if (PyUnicodeDecodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeDecodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeDecodeError_GetObject(exc))) + if (_PyUnicodeError_GetParams(exc, + &obj, &objlen, + &start, &end, &slen, true) < 0) + { return NULL; - p = (const unsigned char*)PyBytes_AS_STRING(object); - res = PyUnicode_New(4 * (end - start), 127); + } + PyObject *res = PyUnicode_New(4 * slen, 127); if (res == NULL) { - Py_DECREF(object); + Py_DECREF(obj); return NULL; } - outp = PyUnicode_1BYTE_DATA(res); - for (i = start; i < end; i++, outp += 4) { - unsigned char c = p[i]; + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + const unsigned char *p = (const unsigned char *)PyBytes_AS_STRING(obj); + for (Py_ssize_t i = start; i < end; i++, outp += 4) { + const unsigned char ch = p[i]; outp[0] = '\\'; outp[1] = 'x'; - outp[2] = Py_hexdigits[(c>>4)&0xf]; - outp[3] = Py_hexdigits[c&0xf]; + outp[2] = Py_hexdigits[(ch >> 4) & 0xf]; + outp[3] = Py_hexdigits[ch & 0xf]; } - assert(_PyUnicode_CheckConsistency(res, 1)); - Py_DECREF(object); + Py_DECREF(obj); return Py_BuildValue("(Nn)", res, end); } - if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - if (PyUnicodeEncodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeEncodeError_GetObject(exc))) - return NULL; - } - else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) { - if (PyUnicodeTranslateError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeTranslateError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeTranslateError_GetObject(exc))) + + if ( + PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError) + || PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError) + ) { + if (_PyUnicodeError_GetParams(exc, + &obj, &objlen, + &start, &end, &slen, false) < 0) + { return NULL; + } } else { wrong_exception_type(exc); return NULL; } - if (end - start > PY_SSIZE_T_MAX / (1+1+8)) - end = start + PY_SSIZE_T_MAX / (1+1+8); - for (i = start, ressize = 0; i < end; ++i) { + // The number of characters that each character 'ch' contributes + // in the result is 1 + 1 + k, where k >= min{t >= 1 | 16^t > ch} + // and will be formatted as "\\" + ('U'|'u'|'x') + HEXDIGITS, + // where the number of hexdigits is either 2, 4, or 8 (not 6). + // Since the Unicode range is below 10^7, we choose k = 8 whence + // each "block" requires at most 1 + 1 + 8 characters. + if (slen > PY_SSIZE_T_MAX / (1 + 1 + 8)) { + end = start + PY_SSIZE_T_MAX / (1 + 1 + 8); + end = Py_MIN(end, objlen); + slen = Py_MAX(0, end - start); + } + + Py_ssize_t ressize = 0; + for (Py_ssize_t i = start; i < end; ++i) { /* object is guaranteed to be "ready" */ - c = PyUnicode_READ_CHAR(object, i); + Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); if (c >= 0x10000) { - ressize += 1+1+8; + ressize += 1 + 1 + 8; } else if (c >= 0x100) { - ressize += 1+1+4; + ressize += 1 + 1 + 4; + } + else { + ressize += 1 + 1 + 2; } - else - ressize += 1+1+2; } - res = PyUnicode_New(ressize, 127); + PyObject *res = PyUnicode_New(ressize, 127); if (res == NULL) { - Py_DECREF(object); + Py_DECREF(obj); return NULL; } - outp = PyUnicode_1BYTE_DATA(res); - for (i = start; i < end; ++i) { - c = PyUnicode_READ_CHAR(object, i); + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + for (Py_ssize_t i = start; i < end; ++i) { + Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); *outp++ = '\\'; if (c >= 0x00010000) { *outp++ = 'U'; - *outp++ = Py_hexdigits[(c>>28)&0xf]; - *outp++ = Py_hexdigits[(c>>24)&0xf]; - *outp++ = Py_hexdigits[(c>>20)&0xf]; - *outp++ = Py_hexdigits[(c>>16)&0xf]; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c >> 28) & 0xf]; + *outp++ = Py_hexdigits[(c >> 24) & 0xf]; + *outp++ = Py_hexdigits[(c >> 20) & 0xf]; + *outp++ = Py_hexdigits[(c >> 16) & 0xf]; + *outp++ = Py_hexdigits[(c >> 12) & 0xf]; + *outp++ = Py_hexdigits[(c >> 8) & 0xf]; } else if (c >= 0x100) { *outp++ = 'u'; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c >> 12) & 0xf]; + *outp++ = Py_hexdigits[(c >> 8) & 0xf]; } - else + else { *outp++ = 'x'; - *outp++ = Py_hexdigits[(c>>4)&0xf]; - *outp++ = Py_hexdigits[c&0xf]; + } + *outp++ = Py_hexdigits[(c >> 4) & 0xf]; + *outp++ = Py_hexdigits[c & 0xf]; } - assert(_PyUnicode_CheckConsistency(res, 1)); - Py_DECREF(object); + Py_DECREF(obj); return Py_BuildValue("(Nn)", res, end); }

1 0

[3.13] gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) (#129217)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/e3b3e01d6a6f43c15890d14f139f381556… commit: e3b3e01d6a6f43c15890d14f139f38155601643a branch: 3.13 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T13:59:19+01:00 summary: [3.13] gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) (#129217) gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) Support calling PyTraceMalloc_Track() and PyTraceMalloc_Untrack() during late Python finalization. * Call _PyTraceMalloc_Fini() later in Python finalization. * Test also PyTraceMalloc_Untrack() without the GIL * PyTraceMalloc_Untrack() now gets the GIL. * Test also PyTraceMalloc_Untrack() in test_tracemalloc_track_race(). (cherry picked from commit 46c7e13c055c218e18b0424efc60965e6a5fe6ea) files: M Lib/test/test_tracemalloc.py M Modules/_testcapi/mem.c M Modules/_testcapimodule.c M Python/pylifecycle.c M Python/tracemalloc.c diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 238ae14b388c76..0220a83d24b428 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1,6 +1,7 @@ import contextlib import os import sys +import textwrap import tracemalloc import unittest from unittest.mock import patch @@ -19,6 +20,7 @@ _testinternalcapi = None +DEFAULT_DOMAIN = 0 EMPTY_STRING_SIZE = sys.getsizeof(b'') INVALID_NFRAME = (-1, 2**30) @@ -1027,8 +1029,8 @@ def track(self, release_gil=False, nframe=1): release_gil) return frames - def untrack(self): - _testcapi.tracemalloc_untrack(self.domain, self.ptr) + def untrack(self, release_gil=False): + _testcapi.tracemalloc_untrack(self.domain, self.ptr, release_gil) def get_traced_memory(self): # Get the traced size in the domain @@ -1070,7 +1072,7 @@ def test_track_already_tracked(self): self.assertEqual(self.get_traceback(), tracemalloc.Traceback(frames)) - def test_untrack(self): + def check_untrack(self, release_gil): tracemalloc.start() self.track() @@ -1078,13 +1080,19 @@ def test_untrack(self): self.assertEqual(self.get_traced_memory(), self.size) # untrack must remove the trace - self.untrack() + self.untrack(release_gil) self.assertIsNone(self.get_traceback()) self.assertEqual(self.get_traced_memory(), 0) # calling _PyTraceMalloc_Untrack() multiple times must not crash - self.untrack() - self.untrack() + self.untrack(release_gil) + self.untrack(release_gil) + + def test_untrack(self): + self.check_untrack(False) + + def test_untrack_without_gil(self): + self.check_untrack(True) def test_stop_track(self): tracemalloc.start() @@ -1110,6 +1118,29 @@ def test_tracemalloc_track_race(self): # gh-128679: Test fix for tracemalloc.stop() race condition _testcapi.tracemalloc_track_race() + def test_late_untrack(self): + code = textwrap.dedent(f""" + from test import support + import tracemalloc + import _testcapi + + class Tracked: + def __init__(self, domain, size): + self.domain = domain + self.ptr = id(self) + self.size = size + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size) + + def __del__(self, untrack=_testcapi.tracemalloc_untrack): + untrack(self.domain, self.ptr, 1) + + domain = {DEFAULT_DOMAIN} + tracemalloc.start() + obj = Tracked(domain, 1024 * 1024) + support.late_deletion(obj) + """) + assert_python_ok("-c", code) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ab4ad934644c38..ecae5ba26226a6 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -557,8 +557,9 @@ tracemalloc_untrack(PyObject *self, PyObject *args) { unsigned int domain; PyObject *ptr_obj; + int release_gil = 0; - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + if (!PyArg_ParseTuple(args, "IO|i", &domain, &ptr_obj, &release_gil)) { return NULL; } void *ptr = PyLong_AsVoidPtr(ptr_obj); @@ -566,7 +567,15 @@ tracemalloc_untrack(PyObject *self, PyObject *args) return NULL; } - int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + } if (res < 0) { PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); return NULL; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 633c53a26246a8..fd6ca815f22efa 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3327,6 +3327,7 @@ static void tracemalloc_track_race_thread(void *data) { PyTraceMalloc_Track(123, 10, 1); + PyTraceMalloc_Untrack(123, 10); PyThread_type_lock lock = (PyThread_type_lock)data; PyThread_release_lock(lock); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5255d137ccca89..ba835ef4c848e8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2156,7 +2156,7 @@ _Py_Finalize(_PyRuntimeState *runtime) /* Disable tracemalloc after all Python objects have been destroyed, so it is possible to use tracemalloc in objects destructor. */ - _PyTraceMalloc_Fini(); + _PyTraceMalloc_Stop(); /* Finalize any remaining import state */ // XXX Move these up to where finalize_modules() is currently. @@ -2209,6 +2209,8 @@ _Py_Finalize(_PyRuntimeState *runtime) finalize_interp_clear(tstate); + _PyTraceMalloc_Fini(); + #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index d86972b7bc0b97..89bc7629933921 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1319,29 +1319,48 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini(). + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int res; if (tracemalloc_config.tracing) { - res = tracemalloc_add_trace(domain, ptr, size); + result = tracemalloc_add_trace(domain, ptr, size); } else { // gh-128679: tracemalloc.stop() was called by another thread - res = -2; + result = -2; } TABLES_UNLOCK(); +done: PyGILState_Release(gil_state); - return res; + return result; } int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { + // Need the GIL to prevent races on the first 'tracing' test + PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini() + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int result; if (tracemalloc_config.tracing) { tracemalloc_remove_trace(domain, ptr); result = 0; @@ -1352,6 +1371,8 @@ PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) } TABLES_UNLOCK(); +done: + PyGILState_Release(gil_state); return result; }

1 0

gh-128863: Deprecate _PyLong_New() function (#129212)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/5c9a63f62c9e56d1576c2dc62c986f2be9… commit: 5c9a63f62c9e56d1576c2dc62c986f2be988729f branch: main author: Sergey B Kirpichev <skirpichev(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T13:50:34+01:00 summary: gh-128863: Deprecate _PyLong_New() function (#129212) files: M Doc/deprecations/c-api-pending-removal-in-3.18.rst M Doc/whatsnew/3.14.rst M Include/cpython/longintrepr.h M Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst M Objects/longobject.c diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 9da3d73c9e1252..361e1a9abf22d7 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -7,6 +7,7 @@ Pending removal in Python 3.18 * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 421375ebaac184..f463ed415d6a20 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1396,6 +1396,7 @@ Deprecated * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 357477b60d9a5a..f0d3d0d2b37e4f 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -100,7 +100,7 @@ struct _longobject { _PyLongValue long_value; }; -PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); +Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); // Return a copy of src. PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); diff --git a/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst index 5e55cfb3c74a4b..7e6a8484b8887c 100644 --- a/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst +++ b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst @@ -5,6 +5,7 @@ Python 3.18: * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. +* :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. diff --git a/Objects/longobject.c b/Objects/longobject.c index 370931e64d3627..89526d5a430304 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -152,8 +152,8 @@ long_normalize(PyLongObject *v) # define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) #endif -PyLongObject * -_PyLong_New(Py_ssize_t size) +static PyLongObject * +long_alloc(Py_ssize_t size) { assert(size >= 0); PyLongObject *result = NULL; @@ -190,6 +190,12 @@ _PyLong_New(Py_ssize_t size) return result; } +PyLongObject * +_PyLong_New(Py_ssize_t size) +{ + return long_alloc(size); +} + PyLongObject * _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) { @@ -197,7 +203,7 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) if (digit_count == 0) { return (PyLongObject *)_PyLong_GetZero(); } - PyLongObject *result = _PyLong_New(digit_count); + PyLongObject *result = long_alloc(digit_count); if (result == NULL) { PyErr_NoMemory(); return NULL; @@ -268,7 +274,7 @@ _PyLong_FromLarge(stwodigits ival) ++ndigits; t >>= PyLong_SHIFT; } - PyLongObject *v = _PyLong_New(ndigits); + PyLongObject *v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, sign, ndigits); @@ -341,7 +347,7 @@ PyLong_FromLong(long ival) } /* Construct output value. */ - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, ival < 0 ? -1 : 1, ndigits); @@ -366,7 +372,7 @@ PyLong_FromLong(long ival) ++ndigits; \ t >>= PyLong_SHIFT; \ } \ - PyLongObject *v = _PyLong_New(ndigits); \ + PyLongObject *v = long_alloc(ndigits); \ if (v == NULL) { \ return NULL; \ } \ @@ -443,7 +449,7 @@ PyLong_FromDouble(double dval) frac = frexp(dval, &expo); /* dval = frac*2**expo; 0.0 <= frac < 1.0 */ assert(expo > 0); ndig = (expo-1) / PyLong_SHIFT + 1; /* Number of 'digits' in result */ - v = _PyLong_New(ndig); + v = long_alloc(ndig); if (v == NULL) return NULL; frac = ldexp(frac, (expo-1) % PyLong_SHIFT + 1); @@ -952,7 +958,7 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, return NULL; } ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v == NULL) return NULL; @@ -1482,7 +1488,7 @@ PyLong_FromLongLong(long long ival) } /* Construct output value. */ - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, ival < 0 ? -1 : 1, ndigits); @@ -1525,7 +1531,7 @@ PyLong_FromSsize_t(Py_ssize_t ival) ++ndigits; t >>= PyLong_SHIFT; } - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, negative ? -1 : 1, ndigits); @@ -2015,7 +2021,7 @@ divrem1(PyLongObject *a, digit n, digit *prem) PyLongObject *z; assert(n > 0 && n <= PyLong_MASK); - z = _PyLong_New(size); + z = long_alloc(size); if (z == NULL) return NULL; *prem = inplace_divrem1(z->long_value.ob_digit, a->long_value.ob_digit, size, n); @@ -2192,7 +2198,7 @@ long_to_decimal_string_internal(PyObject *aa, (10 * PyLong_SHIFT - 33 * _PyLong_DECIMAL_SHIFT); assert(size_a < PY_SSIZE_T_MAX/2); size = 1 + size_a + size_a / d; - scratch = _PyLong_New(size); + scratch = long_alloc(size); if (scratch == NULL) return -1; @@ -2635,7 +2641,7 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int return 0; } n = (digits * bits_per_char + PyLong_SHIFT - 1) / PyLong_SHIFT; - z = _PyLong_New(n); + z = long_alloc(n); if (z == NULL) { *res = NULL; return 0; @@ -2839,7 +2845,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, */ double fsize_z = (double)digits * log_base_BASE[base] + 1.0; if (fsize_z > (double)MAX_LONG_DIGITS) { - /* The same exception as in _PyLong_New(). */ + /* The same exception as in long_alloc(). */ PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); *res = NULL; @@ -2849,7 +2855,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, /* Uncomment next line to test exceedingly rare copy code */ /* size_z = 1; */ assert(size_z > 0); - z = _PyLong_New(size_z); + z = long_alloc(size_z); if (z == NULL) { *res = NULL; return 0; @@ -2912,7 +2918,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, PyLongObject *tmp; /* Extremely rare. Get more space. */ assert(_PyLong_DigitCount(z) == size_z); - tmp = _PyLong_New(size_z + 1); + tmp = long_alloc(size_z + 1); if (tmp == NULL) { Py_DECREF(z); *res = NULL; @@ -3333,12 +3339,12 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) size_v = _PyLong_DigitCount(v1); size_w = _PyLong_DigitCount(w1); assert(size_v >= size_w && size_w >= 2); /* Assert checks by div() */ - v = _PyLong_New(size_v+1); + v = long_alloc(size_v+1); if (v == NULL) { *prem = NULL; return NULL; } - w = _PyLong_New(size_w); + w = long_alloc(size_w); if (w == NULL) { Py_DECREF(v); *prem = NULL; @@ -3360,7 +3366,7 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) at most (and usually exactly) k = size_v - size_w digits. */ k = size_v - size_w; assert(k >= 0); - a = _PyLong_New(k); + a = long_alloc(k); if (a == NULL) { Py_DECREF(w); Py_DECREF(v); @@ -3758,7 +3764,7 @@ x_add(PyLongObject *a, PyLongObject *b) size_a = size_b; size_b = size_temp; } } - z = _PyLong_New(size_a+1); + z = long_alloc(size_a+1); if (z == NULL) return NULL; for (i = 0; i < size_b; ++i) { @@ -3807,7 +3813,7 @@ x_sub(PyLongObject *a, PyLongObject *b) } size_a = size_b = i+1; } - z = _PyLong_New(size_a); + z = long_alloc(size_a); if (z == NULL) return NULL; for (i = 0; i < size_b; ++i) { @@ -3932,7 +3938,7 @@ x_mul(PyLongObject *a, PyLongObject *b) Py_ssize_t size_b = _PyLong_DigitCount(b); Py_ssize_t i; - z = _PyLong_New(size_a + size_b); + z = long_alloc(size_a + size_b); if (z == NULL) return NULL; @@ -4042,9 +4048,9 @@ kmul_split(PyLongObject *n, size_lo = Py_MIN(size_n, size); size_hi = size_n - size_lo; - if ((hi = _PyLong_New(size_hi)) == NULL) + if ((hi = long_alloc(size_hi)) == NULL) return -1; - if ((lo = _PyLong_New(size_lo)) == NULL) { + if ((lo = long_alloc(size_lo)) == NULL) { Py_DECREF(hi); return -1; } @@ -4144,7 +4150,7 @@ k_mul(PyLongObject *a, PyLongObject *b) */ /* 1. Allocate result space. */ - ret = _PyLong_New(asize + bsize); + ret = long_alloc(asize + bsize); if (ret == NULL) goto fail; #ifdef Py_DEBUG /* Fill with trash, to catch reference to uninitialized digits. */ @@ -4294,13 +4300,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) assert(2 * asize <= bsize); /* Allocate result space, and zero it out. */ - ret = _PyLong_New(asize + bsize); + ret = long_alloc(asize + bsize); if (ret == NULL) return NULL; memset(ret->long_value.ob_digit, 0, _PyLong_DigitCount(ret) * sizeof(digit)); /* Successive slices of b are copied into bslice. */ - bslice = _PyLong_New(asize); + bslice = long_alloc(asize); if (bslice == NULL) goto fail; @@ -4766,7 +4772,7 @@ long_true_divide(PyObject *v, PyObject *w) "intermediate overflow during division"); goto error; } - x = _PyLong_New(a_size + shift_digits + 1); + x = long_alloc(a_size + shift_digits + 1); if (x == NULL) goto error; for (i = 0; i < shift_digits; i++) @@ -4780,7 +4786,7 @@ long_true_divide(PyObject *v, PyObject *w) digit rem; /* x = a >> shift */ assert(a_size >= shift_digits); - x = _PyLong_New(a_size - shift_digits); + x = long_alloc(a_size - shift_digits); if (x == NULL) goto error; rem = v_rshift(x->long_value.ob_digit, a->long_value.ob_digit + shift_digits, @@ -5360,7 +5366,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) /* Shifting all the bits of 'a' out gives either -1 or 0. */ return PyLong_FromLong(-a_negative); } - z = _PyLong_New(newsize); + z = long_alloc(newsize); if (z == NULL) { return NULL; } @@ -5475,7 +5481,7 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) newsize = oldsize + wordshift; if (remshift) ++newsize; - z = _PyLong_New(newsize); + z = long_alloc(newsize); if (z == NULL) return NULL; if (_PyLong_IsNegative(a)) { @@ -5590,7 +5596,7 @@ long_bitwise(PyLongObject *a, size_a = _PyLong_DigitCount(a); nega = _PyLong_IsNegative(a); if (nega) { - z = _PyLong_New(size_a); + z = long_alloc(size_a); if (z == NULL) return NULL; v_complement(z->long_value.ob_digit, a->long_value.ob_digit, size_a); @@ -5604,7 +5610,7 @@ long_bitwise(PyLongObject *a, size_b = _PyLong_DigitCount(b); negb = _PyLong_IsNegative(b); if (negb) { - z = _PyLong_New(size_b); + z = long_alloc(size_b); if (z == NULL) { Py_DECREF(a); return NULL; @@ -5648,7 +5654,7 @@ long_bitwise(PyLongObject *a, /* We allow an extra digit if z is negative, to make sure that the final two's complement of z doesn't overflow. */ - z = _PyLong_New(size_z + negz); + z = long_alloc(size_z + negz); if (z == NULL) { Py_DECREF(a); Py_DECREF(b); @@ -5846,7 +5852,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } else { alloc_a = size_a; - c = _PyLong_New(size_a); + c = long_alloc(size_a); if (c == NULL) goto error; } @@ -5862,7 +5868,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } else { alloc_b = size_a; - d = _PyLong_New(size_a); + d = long_alloc(size_a); if (d == NULL) goto error; } @@ -6910,7 +6916,7 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) } assert(digits != NULL); - PyLongObject *obj = _PyLong_New(ndigits); + PyLongObject *obj = long_alloc(ndigits); if (obj == NULL) { goto error; }

1 0

gh-129185: Remove internal TRACE_RAW_MALLOC macro (#129218)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/e579cdb21e25896f85ae1a42820b749846… commit: e579cdb21e25896f85ae1a42820b749846b88de7 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T13:49:35+01:00 summary: gh-129185: Remove internal TRACE_RAW_MALLOC macro (#129218) Always build tracemalloc with PyMem_RawMalloc() hooks. files: M Include/internal/pycore_tracemalloc.h M Python/tracemalloc.c diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 9fed393ac9e60e..3fb0aa133fc41d 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -11,10 +11,6 @@ extern "C" { #include "pycore_hashtable.h" // _Py_hashtable_t -/* Trace memory blocks allocated by PyMem_RawMalloc() */ -#define TRACE_RAW_MALLOC - - struct _PyTraceMalloc_Config { /* Module initialized? Variable protected by the GIL */ @@ -74,9 +70,7 @@ struct _tracemalloc_runtime_state { PyMemAllocatorEx obj; } allocators; -#if defined(TRACE_RAW_MALLOC) PyThread_type_lock tables_lock; -#endif /* Size in bytes of currently traced memory. Protected by TABLES_LOCK(). */ size_t traced_memory; diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 62065e85ac8350..20651016a80ec2 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -33,18 +33,12 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, #define allocators _PyRuntime.tracemalloc.allocators -#if defined(TRACE_RAW_MALLOC) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ -# define tables_lock _PyRuntime.tracemalloc.tables_lock -# define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) -# define TABLES_UNLOCK() PyThread_release_lock(tables_lock) -#else - /* variables are protected by the GIL */ -# define TABLES_LOCK() -# define TABLES_UNLOCK() -#endif +#define tables_lock _PyRuntime.tracemalloc.tables_lock +#define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) +#define TABLES_UNLOCK() PyThread_release_lock(tables_lock) #define DEFAULT_DOMAIN 0 @@ -98,9 +92,6 @@ tracemalloc_error(const char *format, ...) #endif -#if defined(TRACE_RAW_MALLOC) -#define REENTRANT_THREADLOCAL - #define tracemalloc_reentrant_key _PyRuntime.tracemalloc.reentrant_key /* Any non-NULL pointer can be used */ @@ -137,25 +128,6 @@ set_reentrant(int reentrant) } } -#else - -/* TRACE_RAW_MALLOC not defined: variable protected by the GIL */ -static int tracemalloc_reentrant = 0; - -static int -get_reentrant(void) -{ - return tracemalloc_reentrant; -} - -static void -set_reentrant(int reentrant) -{ - assert(reentrant != tracemalloc_reentrant); - tracemalloc_reentrant = reentrant; -} -#endif - static Py_uhash_t hashtable_hash_pyobject(const void *key) @@ -709,7 +681,6 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) } -#ifdef TRACE_RAW_MALLOC static void* tracemalloc_raw_malloc(void *ctx, size_t size) { @@ -729,7 +700,6 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) { return tracemalloc_realloc(1, ctx, ptr, new_size); } -#endif /* TRACE_RAW_MALLOC */ static void @@ -767,20 +737,16 @@ _PyTraceMalloc_Init(void) PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); -#ifdef REENTRANT_THREADLOCAL if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { return _PyStatus_NO_MEMORY(); } -#endif -#if defined(TRACE_RAW_MALLOC) if (tables_lock == NULL) { tables_lock = PyThread_allocate_lock(); if (tables_lock == NULL) { return _PyStatus_NO_MEMORY(); } } -#endif tracemalloc_filenames = hashtable_new(hashtable_hash_pyobject, hashtable_compare_unicode, @@ -826,16 +792,12 @@ tracemalloc_deinit(void) _Py_hashtable_destroy(tracemalloc_tracebacks); _Py_hashtable_destroy(tracemalloc_filenames); -#if defined(TRACE_RAW_MALLOC) if (tables_lock != NULL) { PyThread_free_lock(tables_lock); tables_lock = NULL; } -#endif -#ifdef REENTRANT_THREADLOCAL PyThread_tss_delete(&tracemalloc_reentrant_key); -#endif } @@ -866,7 +828,6 @@ _PyTraceMalloc_Start(int max_nframe) } PyMemAllocatorEx alloc; -#ifdef TRACE_RAW_MALLOC alloc.malloc = tracemalloc_raw_malloc; alloc.calloc = tracemalloc_raw_calloc; alloc.realloc = tracemalloc_raw_realloc; @@ -875,7 +836,6 @@ _PyTraceMalloc_Start(int max_nframe) alloc.ctx = &allocators.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); -#endif alloc.malloc = tracemalloc_malloc_gil; alloc.calloc = tracemalloc_calloc_gil; @@ -916,9 +876,7 @@ _PyTraceMalloc_Stop(void) tracemalloc_config.tracing = 0; /* unregister the hook on memory allocators */ -#ifdef TRACE_RAW_MALLOC PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); -#endif PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj);

1 0

gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/46c7e13c055c218e18b0424efc60965e6a… commit: 46c7e13c055c218e18b0424efc60965e6a5fe6ea branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T12:07:34+01:00 summary: gh-129185: Fix PyTraceMalloc_Untrack() at Python exit (#129191) Support calling PyTraceMalloc_Track() and PyTraceMalloc_Untrack() during late Python finalization. * Call _PyTraceMalloc_Fini() later in Python finalization. * Test also PyTraceMalloc_Untrack() without the GIL * PyTraceMalloc_Untrack() now gets the GIL. * Test also PyTraceMalloc_Untrack() in test_tracemalloc_track_race(). files: M Lib/test/test_tracemalloc.py M Modules/_testcapi/mem.c M Modules/_testcapimodule.c M Python/pylifecycle.c M Python/tracemalloc.c diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 238ae14b388c76..0220a83d24b428 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1,6 +1,7 @@ import contextlib import os import sys +import textwrap import tracemalloc import unittest from unittest.mock import patch @@ -19,6 +20,7 @@ _testinternalcapi = None +DEFAULT_DOMAIN = 0 EMPTY_STRING_SIZE = sys.getsizeof(b'') INVALID_NFRAME = (-1, 2**30) @@ -1027,8 +1029,8 @@ def track(self, release_gil=False, nframe=1): release_gil) return frames - def untrack(self): - _testcapi.tracemalloc_untrack(self.domain, self.ptr) + def untrack(self, release_gil=False): + _testcapi.tracemalloc_untrack(self.domain, self.ptr, release_gil) def get_traced_memory(self): # Get the traced size in the domain @@ -1070,7 +1072,7 @@ def test_track_already_tracked(self): self.assertEqual(self.get_traceback(), tracemalloc.Traceback(frames)) - def test_untrack(self): + def check_untrack(self, release_gil): tracemalloc.start() self.track() @@ -1078,13 +1080,19 @@ def test_untrack(self): self.assertEqual(self.get_traced_memory(), self.size) # untrack must remove the trace - self.untrack() + self.untrack(release_gil) self.assertIsNone(self.get_traceback()) self.assertEqual(self.get_traced_memory(), 0) # calling _PyTraceMalloc_Untrack() multiple times must not crash - self.untrack() - self.untrack() + self.untrack(release_gil) + self.untrack(release_gil) + + def test_untrack(self): + self.check_untrack(False) + + def test_untrack_without_gil(self): + self.check_untrack(True) def test_stop_track(self): tracemalloc.start() @@ -1110,6 +1118,29 @@ def test_tracemalloc_track_race(self): # gh-128679: Test fix for tracemalloc.stop() race condition _testcapi.tracemalloc_track_race() + def test_late_untrack(self): + code = textwrap.dedent(f""" + from test import support + import tracemalloc + import _testcapi + + class Tracked: + def __init__(self, domain, size): + self.domain = domain + self.ptr = id(self) + self.size = size + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size) + + def __del__(self, untrack=_testcapi.tracemalloc_untrack): + untrack(self.domain, self.ptr, 1) + + domain = {DEFAULT_DOMAIN} + tracemalloc.start() + obj = Tracked(domain, 1024 * 1024) + support.late_deletion(obj) + """) + assert_python_ok("-c", code) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ab4ad934644c38..ecae5ba26226a6 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -557,8 +557,9 @@ tracemalloc_untrack(PyObject *self, PyObject *args) { unsigned int domain; PyObject *ptr_obj; + int release_gil = 0; - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + if (!PyArg_ParseTuple(args, "IO|i", &domain, &ptr_obj, &release_gil)) { return NULL; } void *ptr = PyLong_AsVoidPtr(ptr_obj); @@ -566,7 +567,15 @@ tracemalloc_untrack(PyObject *self, PyObject *args) return NULL; } - int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + } if (res < 0) { PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); return NULL; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 996b96bc000450..c405a352ed74a1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3394,6 +3394,7 @@ static void tracemalloc_track_race_thread(void *data) { PyTraceMalloc_Track(123, 10, 1); + PyTraceMalloc_Untrack(123, 10); PyThread_type_lock lock = (PyThread_type_lock)data; PyThread_release_lock(lock); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f6526725d5dccc..52890cfc5df829 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2113,7 +2113,7 @@ _Py_Finalize(_PyRuntimeState *runtime) /* Disable tracemalloc after all Python objects have been destroyed, so it is possible to use tracemalloc in objects destructor. */ - _PyTraceMalloc_Fini(); + _PyTraceMalloc_Stop(); /* Finalize any remaining import state */ // XXX Move these up to where finalize_modules() is currently. @@ -2166,6 +2166,8 @@ _Py_Finalize(_PyRuntimeState *runtime) finalize_interp_clear(tstate); + _PyTraceMalloc_Fini(); + #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index b398ea379e0607..62065e85ac8350 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1256,9 +1256,17 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini(). + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int result; if (tracemalloc_config.tracing) { result = tracemalloc_add_trace_unlocked(domain, ptr, size); } @@ -1268,6 +1276,7 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, } TABLES_UNLOCK(); +done: PyGILState_Release(gil_state); return result; @@ -1277,9 +1286,19 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { + // Need the GIL to prevent races on the first 'tracing' test + PyGILState_STATE gil_state = PyGILState_Ensure(); + int result; + + // gh-129185: Check before TABLES_LOCK() to support calls after + // _PyTraceMalloc_Fini() + if (!tracemalloc_config.tracing) { + result = -2; + goto done; + } + TABLES_LOCK(); - int result; if (tracemalloc_config.tracing) { tracemalloc_remove_trace_unlocked(domain, ptr); result = 0; @@ -1290,6 +1309,8 @@ PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) } TABLES_UNLOCK(); +done: + PyGILState_Release(gil_state); return result; }

1 0

gh-126004: Fix positions handling in `codecs.replace_errors` (#127674)
by picnixz Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/225296cd5b505c180d3f45c355b43d7e1d… commit: 225296cd5b505c180d3f45c355b43d7e1d99d3d5 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-23T11:44:18+01:00 summary: gh-126004: Fix positions handling in `codecs.replace_errors` (#127674) This fixes how `PyCodec_ReplaceErrors` handles the `start` and `end` attributes of `UnicodeError` objects via the `_PyUnicodeError_GetParams` helper. files: A Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst M Lib/test/test_capi/test_codecs.py M Python/codecs.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 3e79dd2f7ca2fa..f57191ddcdbeb4 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -839,7 +839,8 @@ def test_codec_ignore_errors_handler(self): def test_codec_replace_errors_handler(self): handler = _testcapi.codec_replace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors, + safe=True) def test_codec_xmlcharrefreplace_errors_handler(self): handler = _testcapi.codec_xmlcharrefreplace_errors diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst new file mode 100644 index 00000000000000..de70c59ee48eec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.replace_errors` error handler. Patch by Bénédikt +Tran. diff --git a/Python/codecs.c b/Python/codecs.c index 11eaca175abf13..b657dd134a668e 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -702,48 +702,46 @@ PyObject *PyCodec_IgnoreErrors(PyObject *exc) PyObject *PyCodec_ReplaceErrors(PyObject *exc) { - Py_ssize_t start, end, i, len; + Py_ssize_t start, end, slen; if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - PyObject *res; - Py_UCS1 *outp; - if (PyUnicodeEncodeError_GetStart(exc, &start)) + if (_PyUnicodeError_GetParams(exc, NULL, NULL, + &start, &end, &slen, false) < 0) { return NULL; - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; - len = end - start; - res = PyUnicode_New(len, '?'); - if (res == NULL) + } + PyObject *res = PyUnicode_New(slen, '?'); + if (res == NULL) { return NULL; + } assert(PyUnicode_KIND(res) == PyUnicode_1BYTE_KIND); - outp = PyUnicode_1BYTE_DATA(res); - for (i = 0; i < len; ++i) - outp[i] = '?'; + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + memset(outp, '?', sizeof(Py_UCS1) * slen); assert(_PyUnicode_CheckConsistency(res, 1)); return Py_BuildValue("(Nn)", res, end); } else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) { - if (PyUnicodeDecodeError_GetEnd(exc, &end)) + if (_PyUnicodeError_GetParams(exc, NULL, NULL, + NULL, &end, NULL, true) < 0) { return NULL; + } return Py_BuildValue("(Cn)", (int)Py_UNICODE_REPLACEMENT_CHARACTER, end); } else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) { - PyObject *res; - Py_UCS2 *outp; - if (PyUnicodeTranslateError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeTranslateError_GetEnd(exc, &end)) + if (_PyUnicodeError_GetParams(exc, NULL, NULL, + &start, &end, &slen, false) < 0) { return NULL; - len = end - start; - res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER); - if (res == NULL) + } + PyObject *res = PyUnicode_New(slen, Py_UNICODE_REPLACEMENT_CHARACTER); + if (res == NULL) { return NULL; - assert(PyUnicode_KIND(res) == PyUnicode_2BYTE_KIND); - outp = PyUnicode_2BYTE_DATA(res); - for (i = 0; i < len; i++) + } + assert(slen == 0 || PyUnicode_KIND(res) == PyUnicode_2BYTE_KIND); + Py_UCS2 *outp = PyUnicode_2BYTE_DATA(res); + for (Py_ssize_t i = 0; i < slen; ++i) { outp[i] = Py_UNICODE_REPLACEMENT_CHARACTER; + } assert(_PyUnicode_CheckConsistency(res, 1)); return Py_BuildValue("(Nn)", res, end); }

1 0

gh-126004: Fix positions handling in `codecs.xmlcharrefreplace_errors` (#127675)
by picnixz Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/70dcc847df4eb32367859bfa168f057e43… commit: 70dcc847df4eb32367859bfa168f057e43eed11e branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-23T11:42:38+01:00 summary: gh-126004: Fix positions handling in `codecs.xmlcharrefreplace_errors` (#127675) This fixes how `PyCodec_XMLCharRefReplaceErrors` handles the `start` and `end` attributes of `UnicodeError` objects via the `_PyUnicodeError_GetParams` helper. files: A Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst M Lib/test/test_capi/test_codecs.py M Python/codecs.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index a557e35e68915d..3e79dd2f7ca2fa 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -843,7 +843,8 @@ def test_codec_replace_errors_handler(self): def test_codec_xmlcharrefreplace_errors_handler(self): handler = _testcapi.codec_xmlcharrefreplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors, + safe=True) def test_codec_backslashreplace_errors_handler(self): handler = _testcapi.codec_backslashreplace_errors @@ -853,12 +854,12 @@ def test_codec_namereplace_errors_handler(self): handler = _testlimitedcapi.codec_namereplace_errors self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) - def do_test_codec_errors_handler(self, handler, exceptions): + def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): at_least_one = False for exc in exceptions: # See https://github.com/python/cpython/issues/123378 and related # discussion and issues for details. - if self._exception_may_crash(exc): + if not safe and self._exception_may_crash(exc): continue at_least_one = True diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst new file mode 100644 index 00000000000000..60b1c5d8b80793 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.xmlcharrefreplace_errors` error handler. +Patch by Bénédikt Tran. diff --git a/Python/codecs.c b/Python/codecs.c index 2cb3875db35058..11eaca175abf13 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -755,100 +755,113 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc) PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) { - if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - PyObject *restuple; - PyObject *object; - Py_ssize_t i; - Py_ssize_t start; - Py_ssize_t end; - PyObject *res; - Py_UCS1 *outp; - Py_ssize_t ressize; - Py_UCS4 ch; - if (PyUnicodeEncodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeEncodeError_GetObject(exc))) - return NULL; - if (end - start > PY_SSIZE_T_MAX / (2+7+1)) - end = start + PY_SSIZE_T_MAX / (2+7+1); - for (i = start, ressize = 0; i < end; ++i) { - /* object is guaranteed to be "ready" */ - ch = PyUnicode_READ_CHAR(object, i); - if (ch<10) - ressize += 2+1+1; - else if (ch<100) - ressize += 2+2+1; - else if (ch<1000) - ressize += 2+3+1; - else if (ch<10000) - ressize += 2+4+1; - else if (ch<100000) - ressize += 2+5+1; - else if (ch<1000000) - ressize += 2+6+1; - else - ressize += 2+7+1; + if (!PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { + wrong_exception_type(exc); + return NULL; + } + + PyObject *obj; + Py_ssize_t objlen, start, end, slen; + if (_PyUnicodeError_GetParams(exc, + &obj, &objlen, + &start, &end, &slen, false) < 0) + { + return NULL; + } + + // The number of characters that each character 'ch' contributes + // in the result is 2 + k + 1, where k = min{t >= 1 | 10^t > ch} + // and will be formatted as "&#" + DIGITS + ";". Since the Unicode + // range is below 10^7, each "block" requires at most 2 + 7 + 1 + // characters. + if (slen > PY_SSIZE_T_MAX / (2 + 7 + 1)) { + end = start + PY_SSIZE_T_MAX / (2 + 7 + 1); + end = Py_MIN(end, objlen); + slen = Py_MAX(0, end - start); + } + + Py_ssize_t ressize = 0; + for (Py_ssize_t i = start; i < end; ++i) { + /* object is guaranteed to be "ready" */ + Py_UCS4 ch = PyUnicode_READ_CHAR(obj, i); + if (ch < 10) { + ressize += 2 + 1 + 1; } - /* allocate replacement */ - res = PyUnicode_New(ressize, 127); - if (res == NULL) { - Py_DECREF(object); - return NULL; + else if (ch < 100) { + ressize += 2 + 2 + 1; } - outp = PyUnicode_1BYTE_DATA(res); - /* generate replacement */ - for (i = start; i < end; ++i) { - int digits; - int base; - ch = PyUnicode_READ_CHAR(object, i); - *outp++ = '&'; - *outp++ = '#'; - if (ch<10) { - digits = 1; - base = 1; - } - else if (ch<100) { - digits = 2; - base = 10; - } - else if (ch<1000) { - digits = 3; - base = 100; - } - else if (ch<10000) { - digits = 4; - base = 1000; - } - else if (ch<100000) { - digits = 5; - base = 10000; - } - else if (ch<1000000) { - digits = 6; - base = 100000; - } - else { - digits = 7; - base = 1000000; - } - while (digits-->0) { - *outp++ = '0' + ch/base; - ch %= base; - base /= 10; - } - *outp++ = ';'; + else if (ch < 1000) { + ressize += 2 + 3 + 1; + } + else if (ch < 10000) { + ressize += 2 + 4 + 1; + } + else if (ch < 100000) { + ressize += 2 + 5 + 1; + } + else if (ch < 1000000) { + ressize += 2 + 6 + 1; + } + else { + assert(ch < 10000000); + ressize += 2 + 7 + 1; } - assert(_PyUnicode_CheckConsistency(res, 1)); - restuple = Py_BuildValue("(Nn)", res, end); - Py_DECREF(object); - return restuple; } - else { - wrong_exception_type(exc); + + /* allocate replacement */ + PyObject *res = PyUnicode_New(ressize, 127); + if (res == NULL) { + Py_DECREF(obj); return NULL; } + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + /* generate replacement */ + for (Py_ssize_t i = start; i < end; ++i) { + int digits, base; + Py_UCS4 ch = PyUnicode_READ_CHAR(obj, i); + if (ch < 10) { + digits = 1; + base = 1; + } + else if (ch < 100) { + digits = 2; + base = 10; + } + else if (ch < 1000) { + digits = 3; + base = 100; + } + else if (ch < 10000) { + digits = 4; + base = 1000; + } + else if (ch < 100000) { + digits = 5; + base = 10000; + } + else if (ch < 1000000) { + digits = 6; + base = 100000; + } + else { + assert(ch < 10000000); + digits = 7; + base = 1000000; + } + *outp++ = '&'; + *outp++ = '#'; + while (digits-- > 0) { + assert(base >= 1); + *outp++ = '0' + ch / base; + ch %= base; + base /= 10; + } + *outp++ = ';'; + } + assert(_PyUnicode_CheckConsistency(res, 1)); + PyObject *restuple = Py_BuildValue("(Nn)", res, end); + Py_DECREF(obj); + return restuple; } PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)

1 0