Mailman 3 January 2025 - Python-checkins

gh-111178: fix UBSan failures in `Modules/_sre/sre.c` (GH-128250)
by encukou 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/a734c1e304ab459fb34a88f6dd2dbd944a… commit: a734c1e304ab459fb34a88f6dd2dbd944a1b57c9 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-07T12:47:18+01:00 summary: gh-111178: fix UBSan failures in `Modules/_sre/sre.c` (GH-128250) fix UBSan failures for `PatternObject`, `MatchObject`, `TemplateObject`, `ScannerObject` files: M Modules/_sre/sre.c diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 36f542ddb4df2b..d0025dd21e045b 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -395,6 +395,11 @@ static struct PyModuleDef sremodule; static PyObject*pattern_new_match(_sremodulestate *, PatternObject*, SRE_STATE*, Py_ssize_t); static PyObject *pattern_scanner(_sremodulestate *, PatternObject *, PyObject *, Py_ssize_t, Py_ssize_t); +#define _PatternObject_CAST(op) ((PatternObject *)(op)) +#define _MatchObject_CAST(op) ((MatchObject *)(op)) +#define _TemplateObject_CAST(op) ((TemplateObject *)(op)) +#define _ScannerObject_CAST(op) ((ScannerObject *)(op)) + /*[clinic input] module _sre class _sre.SRE_Pattern "PatternObject *" "get_sre_module_state_by_class(tp)->Pattern_Type" @@ -699,8 +704,9 @@ pattern_error(Py_ssize_t status) } static int -pattern_traverse(PatternObject *self, visitproc visit, void *arg) +pattern_traverse(PyObject *op, visitproc visit, void *arg) { + PatternObject *self = _PatternObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->groupindex); Py_VISIT(self->indexgroup); @@ -712,8 +718,9 @@ pattern_traverse(PatternObject *self, visitproc visit, void *arg) } static int -pattern_clear(PatternObject *self) +pattern_clear(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); Py_CLEAR(self->groupindex); Py_CLEAR(self->indexgroup); Py_CLEAR(self->pattern); @@ -724,13 +731,13 @@ pattern_clear(PatternObject *self) } static void -pattern_dealloc(PatternObject* self) +pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); + PatternObject *obj = _PatternObject_CAST(self); + if (obj->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)pattern_clear(self); tp->tp_free(self); @@ -1497,7 +1504,7 @@ _sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, #endif /* Py_DEBUG */ static PyObject * -pattern_repr(PatternObject *obj) +pattern_repr(PyObject *self) { static const struct { const char *name; @@ -1512,6 +1519,8 @@ pattern_repr(PatternObject *obj) {"re.DEBUG", SRE_FLAG_DEBUG}, {"re.ASCII", SRE_FLAG_ASCII}, }; + + PatternObject *obj = _PatternObject_CAST(self); PyObject *result = NULL; PyObject *flag_items; size_t i; @@ -1579,8 +1588,9 @@ PyDoc_STRVAR(pattern_doc, "Compiled regular expression object."); /* PatternObject's 'groupindex' method. */ static PyObject * -pattern_groupindex(PatternObject *self, void *Py_UNUSED(ignored)) +pattern_groupindex(PyObject *op, void *Py_UNUSED(ignored)) { + PatternObject *self = _PatternObject_CAST(op); if (self->groupindex == NULL) return PyDict_New(); return PyDictProxy_New(self->groupindex); @@ -2245,8 +2255,9 @@ _validate(PatternObject *self) /* match methods */ static int -match_traverse(MatchObject *self, visitproc visit, void *arg) +match_traverse(PyObject *op, visitproc visit, void *arg) { + MatchObject *self = _MatchObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->string); Py_VISIT(self->regs); @@ -2255,8 +2266,9 @@ match_traverse(MatchObject *self, visitproc visit, void *arg) } static int -match_clear(MatchObject *self) +match_clear(PyObject *op) { + MatchObject *self = _MatchObject_CAST(op); Py_CLEAR(self->string); Py_CLEAR(self->regs); Py_CLEAR(self->pattern); @@ -2264,10 +2276,9 @@ match_clear(MatchObject *self) } static void -match_dealloc(MatchObject* self) +match_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)match_clear(self); tp->tp_free(self); @@ -2376,8 +2387,9 @@ _sre_SRE_Match_expand_impl(MatchObject *self, PyObject *template) } static PyObject* -match_group(MatchObject* self, PyObject* args) +match_group(PyObject *op, PyObject* args) { + MatchObject *self = _MatchObject_CAST(op); PyObject* result; Py_ssize_t i, size; @@ -2411,8 +2423,9 @@ match_group(MatchObject* self, PyObject* args) } static PyObject* -match_getitem(MatchObject* self, PyObject* name) +match_getitem(PyObject *op, PyObject* name) { + MatchObject *self = _MatchObject_CAST(op); return match_getslice(self, name, Py_None); } @@ -2654,16 +2667,18 @@ PyDoc_STRVAR(match_group_doc, For 0 returns the entire match."); static PyObject * -match_lastindex_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastindex_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->lastindex >= 0) return PyLong_FromSsize_t(self->lastindex); Py_RETURN_NONE; } static PyObject * -match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastgroup_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->pattern->indexgroup && self->lastindex >= 0 && self->lastindex < PyTuple_GET_SIZE(self->pattern->indexgroup)) @@ -2676,8 +2691,9 @@ match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) } static PyObject * -match_regs_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_regs_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->regs) { return Py_NewRef(self->regs); } else @@ -2780,27 +2796,29 @@ pattern_new_match(_sremodulestate* module_state, /* scanner methods (experimental) */ static int -scanner_traverse(ScannerObject *self, visitproc visit, void *arg) +scanner_traverse(PyObject *op, visitproc visit, void *arg) { + ScannerObject *self = _ScannerObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pattern); return 0; } static int -scanner_clear(ScannerObject *self) +scanner_clear(PyObject *op) { + ScannerObject *self = _ScannerObject_CAST(op); Py_CLEAR(self->pattern); return 0; } static void -scanner_dealloc(ScannerObject* self) +scanner_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - state_fini(&self->state); + ScannerObject *scanner = _ScannerObject_CAST(self); + state_fini(&scanner->state); (void)scanner_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -2957,8 +2975,9 @@ pattern_scanner(_sremodulestate *module_state, /* template methods */ static int -template_traverse(TemplateObject *self, visitproc visit, void *arg) +template_traverse(PyObject *op, visitproc visit, void *arg) { + TemplateObject *self = _TemplateObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { @@ -2968,8 +2987,9 @@ template_traverse(TemplateObject *self, visitproc visit, void *arg) } static int -template_clear(TemplateObject *self) +template_clear(PyObject *op) { + TemplateObject *self = _TemplateObject_CAST(op); Py_CLEAR(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { Py_CLEAR(self->items[i].literal); @@ -2978,10 +2998,9 @@ template_clear(TemplateObject *self) } static void -template_dealloc(TemplateObject *self) +template_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)template_clear(self); tp->tp_free(self); @@ -3056,8 +3075,10 @@ expand_template(TemplateObject *self, MatchObject *match) static Py_hash_t -pattern_hash(PatternObject *self) +pattern_hash(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); + Py_hash_t hash, hash2; hash = PyObject_Hash(self->pattern); @@ -3148,7 +3169,7 @@ static PyMethodDef pattern_methods[] = { }; static PyGetSetDef pattern_getset[] = { - {"groupindex", (getter)pattern_groupindex, (setter)NULL, + {"groupindex", pattern_groupindex, NULL, "A dictionary mapping group names to group numbers."}, {NULL} /* Sentinel */ }; @@ -3166,9 +3187,9 @@ static PyMemberDef pattern_members[] = { }; static PyType_Slot pattern_slots[] = { - {Py_tp_dealloc, (destructor)pattern_dealloc}, - {Py_tp_repr, (reprfunc)pattern_repr}, - {Py_tp_hash, (hashfunc)pattern_hash}, + {Py_tp_dealloc, pattern_dealloc}, + {Py_tp_repr, pattern_repr}, + {Py_tp_hash, pattern_hash}, {Py_tp_doc, (void *)pattern_doc}, {Py_tp_richcompare, pattern_richcompare}, {Py_tp_methods, pattern_methods}, @@ -3189,7 +3210,7 @@ static PyType_Spec pattern_spec = { }; static PyMethodDef match_methods[] = { - {"group", (PyCFunction) match_group, METH_VARARGS, match_group_doc}, + {"group", match_group, METH_VARARGS, match_group_doc}, _SRE_SRE_MATCH_START_METHODDEF _SRE_SRE_MATCH_END_METHODDEF _SRE_SRE_MATCH_SPAN_METHODDEF @@ -3204,11 +3225,11 @@ static PyMethodDef match_methods[] = { }; static PyGetSetDef match_getset[] = { - {"lastindex", (getter)match_lastindex_get, (setter)NULL, + {"lastindex", match_lastindex_get, NULL, "The integer index of the last matched capturing group."}, - {"lastgroup", (getter)match_lastgroup_get, (setter)NULL, + {"lastgroup", match_lastgroup_get, NULL, "The name of the last matched capturing group."}, - {"regs", (getter)match_regs_get, (setter)NULL}, + {"regs", match_regs_get, NULL, NULL}, {NULL} };

1 0

gh-111178: fix UBSan failures in `Modules/_abc.c` (GH-128253)
by encukou 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/bcdf654c8a9d8a661e32e6a89f043197de… commit: bcdf654c8a9d8a661e32e6a89f043197de19ad53 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-07T12:45:29+01:00 summary: gh-111178: fix UBSan failures in `Modules/_abc.c` (GH-128253) * fix UBSan failures in `_abc.c` * suppress unused return values files: M Modules/_abc.c diff --git a/Modules/_abc.c b/Modules/_abc.c index 4f4b24b035db4a..d6a953b336025d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -67,6 +67,8 @@ typedef struct { uint64_t _abc_negative_cache_version; } _abc_data; +#define _abc_data_CAST(op) ((_abc_data *)(op)) + static inline uint64_t get_cache_version(_abc_data *impl) { @@ -88,8 +90,9 @@ set_cache_version(_abc_data *impl, uint64_t version) } static int -abc_data_traverse(_abc_data *self, visitproc visit, void *arg) +abc_data_traverse(PyObject *op, visitproc visit, void *arg) { + _abc_data *self = _abc_data_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->_abc_registry); Py_VISIT(self->_abc_cache); @@ -98,8 +101,9 @@ abc_data_traverse(_abc_data *self, visitproc visit, void *arg) } static int -abc_data_clear(_abc_data *self) +abc_data_clear(PyObject *op) { + _abc_data *self = _abc_data_CAST(op); Py_CLEAR(self->_abc_registry); Py_CLEAR(self->_abc_cache); Py_CLEAR(self->_abc_negative_cache); @@ -107,7 +111,7 @@ abc_data_clear(_abc_data *self) } static void -abc_data_dealloc(_abc_data *self) +abc_data_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); PyTypeObject *tp = Py_TYPE(self); @@ -212,7 +216,7 @@ _destroy(PyObject *setweakref, PyObject *objweakref) } static PyMethodDef _destroy_def = { - "_destroy", (PyCFunction) _destroy, METH_O + "_destroy", _destroy, METH_O }; static int @@ -964,7 +968,7 @@ _abcmodule_clear(PyObject *module) static void _abcmodule_free(void *module) { - _abcmodule_clear((PyObject *)module); + (void)_abcmodule_clear((PyObject *)module); } static PyModuleDef_Slot _abcmodule_slots[] = {

1 0

gh-128552: fix refcycles in eager task creation (#128553)
by kumaraditya303 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/61b9811ac6843e22b5896ef96030d421b7… commit: 61b9811ac6843e22b5896ef96030d421b79cd892 branch: main author: Thomas Grainger <tagrain(a)gmail.com> committer: kumaraditya303 <kumaraditya(a)python.org> date: 2025-01-07T11:44:57Z summary: gh-128552: fix refcycles in eager task creation (#128553) files: A Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst M Lib/asyncio/base_events.py M Lib/asyncio/taskgroups.py M Lib/test/test_asyncio/test_taskgroups.py diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 9e6f6e3ee7e3ec..6e6e5aaac15caf 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -477,7 +477,12 @@ def create_task(self, coro, *, name=None, context=None): task.set_name(name) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 9fa772ca9d02cc..8af199d6dcc41a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -205,7 +205,12 @@ def create_task(self, coro, *, name=None, context=None): else: self._tasks.add(task) task.add_done_callback(self._on_task_done) - return task + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task # Since Python 3.8 Tasks propagate all exceptions correctly, # except for KeyboardInterrupt and SystemExit which are diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index c47bf4ec9ed64b..870fa8dbbf2714 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,6 +1,7 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. +import weakref import sys import gc import asyncio @@ -38,7 +39,25 @@ def no_other_refs(): return [coro] -class TestTaskGroup(unittest.IsolatedAsyncioTestCase): +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +(a)contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: async def test_taskgroup_01(self): @@ -832,15 +851,15 @@ async def test_taskgroup_without_parent_task(self): with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - def test_coro_closed_when_tg_closed(self): + async def test_coro_closed_when_tg_closed(self): async def run_coro_after_tg_closes(): async with taskgroups.TaskGroup() as tg: pass coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "is finished"): tg.create_task(coro) - loop = asyncio.get_event_loop() - loop.run_until_complete(run_coro_after_tg_closes()) + + await run_coro_after_tg_closes() async def test_cancelling_level_preserved(self): async def raise_after(t, e): @@ -965,6 +984,30 @@ async def coro_fn(): self.assertIsInstance(exc, _Done) self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + async def test_exception_refcycles_propagate_cancellation_error(self): """Test that TaskGroup deletes propagate_cancellation_error""" tg = asyncio.TaskGroup() @@ -998,5 +1041,16 @@ class MyKeyboardInterrupt(KeyboardInterrupt): self.assertListEqual(gc.get_referrers(exc), no_other_refs()) +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst new file mode 100644 index 00000000000000..83816f775da9c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-18-41-08.gh-issue-128552.fV-f8j.rst @@ -0,0 +1 @@ +Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and :meth:`asyncio.TaskGroup.create_task` holding a reference to the created task if it is eager.

1 0

[3.12] gh-123085: _compile_importlib: Avoid copying sources before compilation (GH-124131) (GH-128581)
by encukou 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/e758eb301804936a5339e6e4a2af99cebc… commit: e758eb301804936a5339e6e4a2af99cebc03fa83 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-07T12:28:03+01:00 summary: [3.12] gh-123085: _compile_importlib: Avoid copying sources before compilation (GH-124131) (GH-128581) gh-123085: _compile_importlib: Avoid copying sources before compilation (GH-124131) (cherry picked from commit 42c8b0556c02d29e32f4c7c95e7128a343716250) Co-authored-by: Petr Viktorin <encukou(a)gmail.com> Co-authored-by: Jason R. Coombs <jaraco(a)jaraco.com> files: M Lib/test/test_importlib/resources/test_files.py diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index d34ea51b150f1e..489cc95d1d6fa5 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -150,18 +150,17 @@ def create_zip_from_directory(source_dir, zip_filename): def _compile_importlib(self): """ Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. """ bin_site = self.fixtures.enter_context(os_helper.temp_dir()) c_resources = pathlib.Path(bin_site, 'c_resources') sources = pathlib.Path(resources.__file__).parent - shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__']) - - for dirpath, _, filenames in os.walk(c_resources): - for filename in filenames: - source_path = pathlib.Path(dirpath) / filename - cfile = source_path.with_suffix('.pyc') - py_compile.compile(source_path, cfile) - pathlib.Path.unlink(source_path) + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) def test_implicit_files_with_compiled_importlib(self):

1 0

gh-128302: Fix bugs in xml.dom.xmlbuilder (GH-128284)
by serhiy-storchaka 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/6ea04da27036eaa69d65150148bb8c537d… commit: 6ea04da27036eaa69d65150148bb8c537d9beacf branch: main author: Stephen Morton <git(a)tungol.org> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2025-01-07T12:40:41+02:00 summary: gh-128302: Fix bugs in xml.dom.xmlbuilder (GH-128284) * Allow DOMParser.parse() to correctly handle DOMInputSource instances that only have a systemId attribute set. * Fix DOMEntityResolver.resolveEntity(), which was broken by the Python 3.0 transition. * Add Lib/test/test_xml_dom_xmlbuilder.py with few tests. files: A Lib/test/test_xml_dom_xmlbuilder.py A Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst A Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst M Lib/xml/dom/xmlbuilder.py diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py new file mode 100644 index 00000000000000..5f5f2eb328df9f --- /dev/null +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -0,0 +1,88 @@ +import io +import unittest +from http import client +from test.test_httplib import FakeSocket +from unittest import mock +from xml.dom import getDOMImplementation, minidom, xmlbuilder + +SMALL_SAMPLE = b"""<?xml version="1.0"?> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books"> + +<title>Introduction to XSL</title> +<hr/> +<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p> +</html>""" + + +class XMLBuilderTest(unittest.TestCase): + def test_entity_resolver(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + resolver = xmlbuilder.DOMEntityResolver() + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + source = resolver.resolveEntity(None, "http://example.com/2000/svg") + + self.assertIsInstance(source, xmlbuilder.DOMInputSource) + self.assertIsNone(source.publicId) + self.assertEqual(source.systemId, "http://example.com/2000/svg") + self.assertEqual(source.baseURI, "http://example.com/2000/") + self.assertEqual(source.encoding, "utf-8") + self.assertIs(source.byteStream, response) + + self.assertIsNone(source.characterStream) + self.assertIsNone(source.stringData) + + def test_builder(self): + imp = getDOMImplementation() + self.assertIsInstance(imp, xmlbuilder.DOMImplementationLS) + + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + self.assertIsInstance(builder, xmlbuilder.DOMBuilder) + + def test_parse_uri(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + + imp = getDOMImplementation() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + document = builder.parseURI("http://example.com/2000/svg") + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) + + def test_parse_with_systemId(self): + response = io.BytesIO(SMALL_SAMPLE) + + with mock.patch("urllib.request.urlopen") as mock_open: + mock_open.return_value = response + + imp = getDOMImplementation() + source = imp.createDOMInputSource() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + source.systemId = "http://example.com/2000/svg" + document = builder.parse(source) + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index 8a200263497b89..a8852625a2f9a2 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -189,7 +189,7 @@ def parse(self, input): options.filter = self.filter options.errorHandler = self.errorHandler fp = input.byteStream - if fp is None and options.systemId: + if fp is None and input.systemId: import urllib.request fp = urllib.request.urlopen(input.systemId) return self._parse_bytestream(fp, options) @@ -247,10 +247,12 @@ def _create_opener(self): def _guess_media_encoding(self, source): info = source.byteStream.info() - if "Content-Type" in info: - for param in info.getplist(): - if param.startswith("charset="): - return param.split("=", 1)[1].lower() + # import email.message + # assert isinstance(info, email.message.Message) + charset = info.get_param('charset') + if charset is not None: + return charset.lower() + return None class DOMInputSource(object): diff --git a/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst b/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst new file mode 100644 index 00000000000000..56e2fe6f85f4bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-27-16-28-57.gh-issue-128302.2GMvyl.rst @@ -0,0 +1,3 @@ +Allow :meth:`!xml.dom.xmlbuilder.DOMParser.parse` to correctly handle +:class:`!xml.dom.xmlbuilder.DOMInputSource` instances that only have a +:attr:`!systemId` attribute set. diff --git a/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst b/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst new file mode 100644 index 00000000000000..98c07297b06f8a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-29-13-49-46.gh-issue-128302.psRpPN.rst @@ -0,0 +1,2 @@ +Fix :meth:`!xml.dom.xmlbuilder.DOMEntityResolver.resolveEntity`, which was +broken by the Python 3.0 transition.

1 0

Add `.. availability:: Windows` directive to COM-related function prototype (GH-127436)
by encukou 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/145276a072ae058bac42ee43a4235cd4ed… commit: 145276a072ae058bac42ee43a4235cd4eda2726b branch: main author: Jun Komoda <45822440+junkmd(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2025-01-07T09:36:27+01:00 summary: Add `.. availability:: Windows` directive to COM-related function prototype (GH-127436) Add a directive to prototype that "returns a foreign function that will call a COM method" files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 398cb92bac809a..f25bf94417c198 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1812,6 +1812,8 @@ different ways, depending on the type and number of the parameters in the call: the COM interface as first argument, in addition to those parameters that are specified in the :attr:`!argtypes` tuple. + .. availability:: Windows + The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above.

1 0

[3.12] gh-108202: Document calendar.Calendar.firstweekday (GH-128566) (#128580)
by erlend-aasland 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/adddbc3de95154c7ccc3af80d9f4d57abf… commit: adddbc3de95154c7ccc3af80d9f4d57abf1c35c2 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2025-01-07T08:06:52Z summary: [3.12] gh-108202: Document calendar.Calendar.firstweekday (GH-128566) (#128580) (cherry picked from commit b3cbd8f1b58b9e71df20bdda9960e7d3190c509e) Co-authored-by: Erlend E. Aasland <erlend(a)python.org> files: M Doc/library/calendar.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 9106092eb123d7..88a90f420e9887 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -38,21 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. .. method:: getfirstweekday() - Return an :class:`int` for the current first weekday (0-6). + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` where Monday is 0 and Sunday is 6. + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month)

1 0

[3.13] gh-108202: Document calendar.Calendar.firstweekday (GH-128566) (#128579)
by erlend-aasland 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/8f3e5554ba09ec14b5249a3b03a592b12c… commit: 8f3e5554ba09ec14b5249a3b03a592b12cf5f53f branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2025-01-07T08:05:09Z summary: [3.13] gh-108202: Document calendar.Calendar.firstweekday (GH-128566) (#128579) (cherry picked from commit b3cbd8f1b58b9e71df20bdda9960e7d3190c509e) Co-authored-by: Erlend E. Aasland <erlend(a)python.org> files: M Doc/library/calendar.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 226af250dc7e6c..086bfcf2e3a909 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -38,21 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. .. method:: getfirstweekday() - Return an :class:`int` for the current first weekday (0-6). + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` where Monday is 0 and Sunday is 6. + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month)

1 0

gh-108202: Document calendar.Calendar.firstweekday (#128566)
by erlend-aasland 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/b3cbd8f1b58b9e71df20bdda9960e7d319… commit: b3cbd8f1b58b9e71df20bdda9960e7d3190c509e branch: main author: Erlend E. Aasland <erlend(a)python.org> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2025-01-07T08:59:24+01:00 summary: gh-108202: Document calendar.Calendar.firstweekday (#128566) files: M Doc/library/calendar.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index dadd301ebc795b..f25930eb0d873e 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -38,21 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. .. method:: getfirstweekday() - Return an :class:`int` for the current first weekday (0-6). + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` where Monday is 0 and Sunday is 6. + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month)

1 0

[3.13] gh-128146: Exclude os/log.h import on older macOS versions. (GH-128165) (#128575)
by freakboy3742 07 Jan '25

07 Jan '25

https://github.com/python/cpython/commit/bbe0b33d2a6e73fb488a72a61329dafd97… commit: bbe0b33d2a6e73fb488a72a61329dafd974cbba6 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: freakboy3742 <russell(a)keith-magee.com> date: 2025-01-07T05:36:00Z summary: [3.13] gh-128146: Exclude os/log.h import on older macOS versions. (GH-128165) (#128575) gh-128146: Exclude os/log.h import on older macOS versions. (GH-128165) Reworks the handling of Apple system log handling to account for older macOS versions that don't provide os-log. (cherry picked from commit e837a1f71e832ce8f551a6fac05e346f654457e0) Co-authored-by: Russell Keith-Magee <russell(a)keith-magee.com> files: A Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst M Python/pylifecycle.c diff --git a/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst b/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst new file mode 100644 index 00000000000000..dfe659294c712e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-12-22-08-54-30.gh-issue-127592.iyuFCC.rst @@ -0,0 +1,2 @@ +Usage of the unified Apple System Log APIs was disabled when the minimum +macOS version is earlier than 10.12. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d23cfc62d0fe9a..d545123a8a7e22 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -44,8 +44,25 @@ #if defined(__APPLE__) # include <AvailabilityMacros.h> +# include <TargetConditionals.h> # include <mach-o/loader.h> -# include <os/log.h> +// The os_log unified logging APIs were introduced in macOS 10.12, iOS 10.0, +// tvOS 10.0, and watchOS 3.0; +# if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +# define HAS_APPLE_SYSTEM_LOG 1 +# elif defined(TARGET_OS_OSX) && TARGET_OS_OSX +# if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 +# define HAS_APPLE_SYSTEM_LOG 1 +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif + +# if HAS_APPLE_SYSTEM_LOG +# include <os/log.h> +# endif #endif #ifdef HAVE_SIGNAL_H @@ -75,7 +92,7 @@ static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ static PyStatus init_android_streams(PyThreadState *tstate); #endif -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG static PyStatus init_apple_streams(PyThreadState *tstate); #endif static void wait_for_thread_shutdown(PyThreadState *tstate); @@ -1258,7 +1275,7 @@ init_interp_main(PyThreadState *tstate) return status; } #endif -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG if (config->use_system_logger) { status = init_apple_streams(tstate); if (_PyStatus_EXCEPTION(status)) { @@ -2933,7 +2950,7 @@ init_android_streams(PyThreadState *tstate) #endif // __ANDROID__ -#if defined(__APPLE__) +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG static PyObject * apple_log_write_impl(PyObject *self, PyObject *args) @@ -2944,14 +2961,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) return NULL; } - // Call the underlying Apple logging API. The os_log unified logging APIs - // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; - // this call is a no-op on older versions. - #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); - #endif Py_RETURN_NONE; } @@ -2986,7 +2998,6 @@ init_apple_streams(PyThreadState *tstate) if (result == NULL) { goto error; } - goto done; error: @@ -3000,7 +3011,7 @@ init_apple_streams(PyThreadState *tstate) return status; } -#endif // __APPLE__ +#endif // __APPLE__ && HAS_APPLE_SYSTEM_LOG static void

1 0