https://github.com/python/cpython/commit/c83013c92dfdc77b87a523b736b76d5abb8... commit: c83013c92dfdc77b87a523b736b76d5abb8ede2a branch: main author: Serhiy Storchaka <storchaka@gmail.com> committer: serhiy-storchaka <storchaka@gmail.com> date: 2026-08-31T21:27:24Z summary: gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734) ElementTree.parse() with a text file mis-decoded the text in the C implementation: _parse_whole() encoded it as UTF-8, but left expat to honor the encoding declared in the document. It now overrides the encoding, as XMLParser.feed() already does for str data. files: A Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst M Lib/test/test_xml_etree.py M Modules/_elementtree.c diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 2af2d1fd64520b..fb35bb6a5f442f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1068,6 +1068,34 @@ def bxml(encoding, body=''): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + def test_parse_text_source(self): + # gh-99064: The encoding declared in the document does not apply + # to a source which is already decoded. + def check(encoding, body): + xml = (f"<?xml version='1.0' encoding='{encoding}'?>" + f"<xml>{body}</xml>") + with self.subTest(encoding=encoding): + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, + body) + # the same with an explicitly created parser + self.assertEqual( + ET.parse(io.StringIO(xml), ET.XMLParser()).getroot().text, + body) + check("ascii", 'a') + check("iso-8859-1", '\xbd') + check("iso-8859-15", '\u20ac') + check("cp437", '\u221a') + check("utf-8", '\u4e2d') + # not ASCII compatible, unsupported for a bytes source + check("utf-16", '\u4e2d') + check("utf-32", '\u4e2d') + + def test_parse_text_source_multiple_chunks(self): + # the encoding is overridden before the first chunk is parsed + body = '\xe4' * 100_000 + xml = "<?xml version='1.0' encoding='ISO-8859-1'?><xml>%s</xml>" % body + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + @support.subTests('sample,exception', [ (b'<x> \xa1</x>', UnicodeDecodeError), # crashed (b'<x> \xa1</x', UnicodeDecodeError), # crashed diff --git a/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst new file mode 100644 index 00000000000000..37a8c0b620310a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst @@ -0,0 +1,5 @@ +Fix :func:`xml.etree.ElementTree.parse` with a text file or other source +of :class:`str` data in the C implementation. +The encoding declared in the document was applied to the already decoded +text, which produced mojibake. It is now ignored, as when parsing with +:meth:`!XMLParser.feed` or :func:`~xml.etree.ElementTree.fromstring`. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f827274eeffba8..a49811a338e625 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4084,6 +4084,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, /* read from open file object */ elementtreestate *st = self->state; + int first = 1; for (;;) { buffer = PyObject_CallFunction(reader, "i", 64*1024); @@ -4100,6 +4101,11 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, Py_DECREF(buffer); break; } + if (first) { + /* The text is already decoded, the encoding declared in the + document does not apply to it. Return code ignored. */ + (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); + } temp = PyUnicode_AsEncodedString(buffer, "utf-8", "surrogatepass"); Py_DECREF(buffer); if (!temp) { @@ -4123,6 +4129,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, res = expat_parse( st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), 0); + first = 0; Py_DECREF(buffer);