Mailman 3 August 2026 - Python-checkins

[3.14] gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734) (GH-156751)
by serhiy-storchaka Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/4e8bce4d547eab49410ae9fb0fcf7b67c2… commit: 4e8bce4d547eab49410ae9fb0fcf7b67c2349056 branch: 3.14 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2026-08-31T22:19:05Z summary: [3.14] gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734) (GH-156751) ElementTree.parse() with a text file mis-decoded the text in the C implementation: _parse_whole() encoded it as UTF-8, but left expat to honor the encoding declared in the document. It now overrides the encoding, as XMLParser.feed() already does for str data. (cherry picked from commit c83013c92dfdc77b87a523b736b76d5abb8ede2a) files: A Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst M Lib/test/test_xml_etree.py M Modules/_elementtree.c diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 6da7c34dc4d3e1..ed6cfd57ddc7c1 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1024,6 +1024,34 @@ def bxml(encoding, body=''): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + def test_parse_text_source(self): + # gh-99064: The encoding declared in the document does not apply + # to a source which is already decoded. + def check(encoding, body): + xml = (f"<?xml version='1.0' encoding='{encoding}'?>" + f"<xml>{body}</xml>") + with self.subTest(encoding=encoding): + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, + body) + # the same with an explicitly created parser + self.assertEqual( + ET.parse(io.StringIO(xml), ET.XMLParser()).getroot().text, + body) + check("ascii", 'a') + check("iso-8859-1", '\xbd') + check("iso-8859-15", '\u20ac') + check("cp437", '\u221a') + check("utf-8", '\u4e2d') + # not ASCII compatible, unsupported for a bytes source + check("utf-16", '\u4e2d') + check("utf-32", '\u4e2d') + + def test_parse_text_source_multiple_chunks(self): + # the encoding is overridden before the first chunk is parsed + body = '\xe4' * 100_000 + xml = "<?xml version='1.0' encoding='ISO-8859-1'?><xml>%s</xml>" % body + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + def test_methods(self): # Test serialization methods. diff --git a/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst new file mode 100644 index 00000000000000..37a8c0b620310a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst @@ -0,0 +1,5 @@ +Fix :func:`xml.etree.ElementTree.parse` with a text file or other source +of :class:`str` data in the C implementation. +The encoding declared in the document was applied to the already decoded +text, which produced mojibake. It is now ignored, as when parsing with +:meth:`!XMLParser.feed` or :func:`~xml.etree.ElementTree.fromstring`. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 6d867d2632cc9a..e8546a5e88ce82 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4072,6 +4072,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, /* read from open file object */ elementtreestate *st = self->state; + int first = 1; for (;;) { buffer = PyObject_CallFunction(reader, "i", 64*1024); @@ -4088,6 +4089,11 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, Py_DECREF(buffer); break; } + if (first) { + /* The text is already decoded, the encoding declared in the + document does not apply to it. Return code ignored. */ + (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); + } temp = PyUnicode_AsEncodedString(buffer, "utf-8", "surrogatepass"); Py_DECREF(buffer); if (!temp) { @@ -4111,6 +4117,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, res = expat_parse( st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), 0); + first = 0; Py_DECREF(buffer);

1 0

gh-155981: Store refleak deltas in arrays (gh-55982)
by nascheme Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/b38e073f1be8fe40af991c044a791cefff… commit: b38e073f1be8fe40af991c044a791cefff098f0d branch: main author: Neil Schemenauer <nas-github(a)arctrix.com> committer: nascheme <nas-github(a)arctrix.com> date: 2026-08-31T14:54:35-07:00 summary: gh-155981: Store refleak deltas in arrays (gh-55982) Store per-run deltas in array objects rather than lists of pooled integers. Large, unique deltas could otherwise grow int_pool and make the refleak checker report its own retained integers as reference leaks. files: M Lib/test/libregrtest/refleak.py diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index e7da17e500ead96..ffb8438d1b0278f 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,6 +1,7 @@ import os import sys import warnings +from array import array from inspect import isabstract from typing import Any import linecache @@ -100,24 +101,20 @@ def runtest_refleak(test_name, test_func, for obj in ByteString.__subclasses__() + [ByteString]: # type: ignore[attr-defined] abcs[obj] = _get_dump(obj)[0] - # bpo-31217: Integer pool to get a single integer object for the same - # value. The pool is used to prevent false alarm when checking for memory - # block leaks. Fill the pool with values in -1000..1000 which are the most - # common (reference, memory block, file descriptor) differences. - int_pool = {value: value for value in range(-1000, 1000)} - def get_pooled_int(value): - return int_pool.setdefault(value, value) - warmups = hunt_refleak.warmups runs = hunt_refleak.runs filename = hunt_refleak.filename repcount = warmups + runs - # Pre-allocate to ensure that the loop doesn't allocate anything new + # Pre-allocate to ensure that the loop doesn't allocate anything new. + # Store the deltas as raw values in arrays rather than as int objects in + # lists: each unique delta stored as an object would live until the end of + # the loop and show up in the following repetition's reference and memory + # block deltas (gh-75400, gh-155981). rep_range = list(range(repcount)) - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - fd_deltas = [0] * repcount + rc_deltas = array('q', [0]) * repcount + alloc_deltas = array('q', [0]) * repcount + fd_deltas = array('q', [0]) * repcount getallocatedblocks = sys.getallocatedblocks gettotalrefcount = sys.gettotalrefcount getunicodeinternedsize = sys.getunicodeinternedsize @@ -161,12 +158,11 @@ def get_pooled_int(value): rc_after = gettotalrefcount() fd_after = fd_count() - rc_deltas[i] = get_pooled_int(rc_after - rc_before) - alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) - fd_deltas[i] = get_pooled_int(fd_after - fd_before) + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + fd_deltas[i] = fd_after - fd_before if not quiet: - # use max, not sum, so total_leaks is one of the pooled ints total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) if total_leaks <= 0: symbol = '.' @@ -212,13 +208,13 @@ def check_fd_deltas(deltas): return any(deltas) failed = False - for deltas, item_name, checker in [ + for raw_deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), (alloc_deltas, 'memory blocks', check_rc_deltas), (fd_deltas, 'file descriptors', check_fd_deltas) ]: - # ignore warmup runs - deltas = deltas[warmups:] + # ignore warmup runs; convert to a list for reporting + deltas = list(raw_deltas[warmups:]) failing = checker(deltas) suspicious = any(deltas) if failing or suspicious:

1 0

gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734)
by serhiy-storchaka Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/c83013c92dfdc77b87a523b736b76d5abb… commit: c83013c92dfdc77b87a523b736b76d5abb8ede2a branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2026-08-31T21:27:24Z summary: gh-99064: Ignore the encoding declaration when parsing decoded text (GH-156734) ElementTree.parse() with a text file mis-decoded the text in the C implementation: _parse_whole() encoded it as UTF-8, but left expat to honor the encoding declared in the document. It now overrides the encoding, as XMLParser.feed() already does for str data. files: A Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst M Lib/test/test_xml_etree.py M Modules/_elementtree.c diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 2af2d1fd64520b..fb35bb6a5f442f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1068,6 +1068,34 @@ def bxml(encoding, body=''): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + def test_parse_text_source(self): + # gh-99064: The encoding declared in the document does not apply + # to a source which is already decoded. + def check(encoding, body): + xml = (f"<?xml version='1.0' encoding='{encoding}'?>" + f"<xml>{body}</xml>") + with self.subTest(encoding=encoding): + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, + body) + # the same with an explicitly created parser + self.assertEqual( + ET.parse(io.StringIO(xml), ET.XMLParser()).getroot().text, + body) + check("ascii", 'a') + check("iso-8859-1", '\xbd') + check("iso-8859-15", '\u20ac') + check("cp437", '\u221a') + check("utf-8", '\u4e2d') + # not ASCII compatible, unsupported for a bytes source + check("utf-16", '\u4e2d') + check("utf-32", '\u4e2d') + + def test_parse_text_source_multiple_chunks(self): + # the encoding is overridden before the first chunk is parsed + body = '\xe4' * 100_000 + xml = "<?xml version='1.0' encoding='ISO-8859-1'?><xml>%s</xml>" % body + self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body) + @support.subTests('sample,exception', [ (b'<x> \xa1</x>', UnicodeDecodeError), # crashed (b'<x> \xa1</x', UnicodeDecodeError), # crashed diff --git a/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst new file mode 100644 index 00000000000000..37a8c0b620310a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-31-20-30-00.gh-issue-99064.Rt4mZ9.rst @@ -0,0 +1,5 @@ +Fix :func:`xml.etree.ElementTree.parse` with a text file or other source +of :class:`str` data in the C implementation. +The encoding declared in the document was applied to the already decoded +text, which produced mojibake. It is now ignored, as when parsing with +:meth:`!XMLParser.feed` or :func:`~xml.etree.ElementTree.fromstring`. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f827274eeffba8..a49811a338e625 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4084,6 +4084,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, /* read from open file object */ elementtreestate *st = self->state; + int first = 1; for (;;) { buffer = PyObject_CallFunction(reader, "i", 64*1024); @@ -4100,6 +4101,11 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, Py_DECREF(buffer); break; } + if (first) { + /* The text is already decoded, the encoding declared in the + document does not apply to it. Return code ignored. */ + (void)EXPAT(st, SetEncoding)(self->parser, "utf-8"); + } temp = PyUnicode_AsEncodedString(buffer, "utf-8", "surrogatepass"); Py_DECREF(buffer); if (!temp) { @@ -4123,6 +4129,7 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject *self, res = expat_parse( st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer), 0); + first = 0; Py_DECREF(buffer);

1 0

[3.14] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156732)
by StanFromIreland Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/97aaaf986c8e66776595e3081ef3d9bb4d… commit: 97aaaf986c8e66776595e3081ef3d9bb4d711c45 branch: 3.14 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: StanFromIreland <stan(a)python.org> date: 2026-08-31T19:09:28Z summary: [3.14] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156732) (cherry picked from commit 287b7cffb79d443614be51b48eaf085d663aeecd) Co-authored-by: Stan Ulbrych <stan(a)python.org> files: A Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst D Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst M Misc/sbom.spdx.json M Modules/expat/expat.h M Modules/expat/internal.h M Modules/expat/refresh.sh M Modules/expat/xmlparse.c M Modules/expat/xmltok.h diff --git a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst similarity index 59% rename from Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst rename to Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst index 439366c8633e824..3dda6055f307d94 100644 --- a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst +++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst @@ -1,2 +1 @@ -Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.3 -for the fix to :cve:`2026-72522`. +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.4. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 18564ce624d314d..6d8ccc06551eb3d 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7baecf6e04769cfb0c5ce2a6e3241e3a0bb8c9e9" + "checksumValue": "12dffaa4a67cbe308643dbec7ffc1b4fd38abbde" }, { "algorithm": "SHA256", - "checksumValue": "d3f19ed52dc975741ecc5a0fc553f910a241d60c76fa4621356d0cdb0490ca28" + "checksumValue": "0e912e25375e213b6e4ff90d554e0a0e037e6f0c20dfa734d366e5bdff289f20" } ], "fileName": "Modules/expat/expat.h" @@ -104,11 +104,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "476a11d9872f8f38844e398c5486ad183ffe2dcf" + "checksumValue": "4afd563c90edd6b4aa5abedcd3df5df023668d26" }, { "algorithm": "SHA256", - "checksumValue": "89f661fa3fa5f7892d83a13ecd685a56aace3fe740abce88a863031114ee2cef" + "checksumValue": "beb7211c800d827743bd3d6ddb86538302d6c51180be6d3b61a1c315e061762d" } ], "fileName": "Modules/expat/internal.h" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0939e3fe0ebb21a5b8ed9d9fdd33cde75ee5658a" + "checksumValue": "b9e4628f37353a7eec8a98c26ebb88d7e2d48971" }, { "algorithm": "SHA256", - "checksumValue": "da48375e85bdc2f97da4445169aafc0b363f150a1a8275dd417e6d84cfc3e443" + "checksumValue": "9afa5cb812283750f1970e230ba392201bac46240abf51ab65e5090e93ca34a6" } ], "fileName": "Modules/expat/xmlparse.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e4bf167669dddff38269486f33eccb0fde0c7ca" + "checksumValue": "e9a5972f664c1c530443ddd8b7900e406e3ca02a" }, { "algorithm": "SHA256", - "checksumValue": "20013b75027e04e324452a002100076e30ec20e0f28b318f392317f99a4c4115" + "checksumValue": "41a6cef659ef1da9ee732304332c4134afe4b63571442de77eaf9918abf9e5df" } ], "fileName": "Modules/expat/xmltok.h" @@ -1772,14 +1772,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" + "checksumValue": "b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.…", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.…", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1787,7 +1787,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.3" + "versionInfo": "2.8.4" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index dbebd985a652ac7..b296be9dbad2dc6 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1096,7 +1096,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 3 +# define XML_MICRO_VERSION 4 # ifdef __cplusplus } diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 7e67d2e378c5243..6311028e94b8f5f 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -33,6 +33,7 @@ Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild(a)sony.com> Copyright (c) 2024 Taichi Haradaguchi <20001722(a)ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka <mattheww(a)simba.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 6c4ef37785152c9..d27ea5404fcc157 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_3" -expected_libexpat_version="2.8.3" -expected_libexpat_sha256="22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" +expected_libexpat_tag="R_2_8_4" +expected_libexpat_version="2.8.4" +expected_libexpat_sha256="b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 4fa61bca8c16293..9a05da21d5a7fd3 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ee5f82c3ffd57c5224394ba46f348dbce466d34d6c925a527ae46b1cfe6adf1d (2.8.3+) +/* 13c4e8da8fccffb0e8e599684e0d447ad14c1bb0b48792cf5dd77d8712301871 (2.8.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -51,6 +51,9 @@ Copyright (c) 2026 Kartik Kenchi <netliomax25(a)gmail.com> Copyright (c) 2026 Haris Hussain <hextheshadow0x(a)gmail.com> Copyright (c) 2026 Evgeny Kotkov <kotkov(a)apache.org> + Copyright (c) 2026 Darren Carreras <carrerasdarren(a)gmail.com> + Copyright (c) 2026 Alberto Maschietto <albertomaschietto9(a)gmail.com> + Copyright (c) 2026 Zeyou Liu <zeyouliu(a)tencent.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -330,7 +333,7 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; + bool open; XML_Bool hasMore; /* true if entity has not been completely processed */ /* An entity can be open while being already completely processed (hasMore == XML_FALSE). The reason is the delayed closing of entities until their inner @@ -381,6 +384,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -394,7 +413,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -579,6 +598,8 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); +static NAMED *lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, + size_t nameLen, size_t createSize); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); @@ -755,6 +776,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -1177,6 +1200,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1524,6 +1566,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1604,7 +1647,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); @@ -1915,7 +1958,7 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); FREE(parser, parser); } @@ -2739,7 +2782,7 @@ XML_GetCurrentLineNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -2756,7 +2799,7 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -3410,9 +3453,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3837,8 +3880,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3951,11 +3994,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4046,8 +4092,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4946,25 +4992,34 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } @@ -6092,7 +6147,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6101,11 +6156,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6429,7 +6484,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; entity->hasMore = XML_TRUE; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); @@ -6520,7 +6575,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6598,7 +6653,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6894,7 +6949,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (entity->systemId) { if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6903,12 +6958,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7058,7 +7113,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7239,7 +7294,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7275,11 +7330,24 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7480,7 +7548,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7597,7 +7665,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); @@ -7639,7 +7707,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7732,8 +7800,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7766,11 +7834,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7867,19 +7946,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } @@ -7897,18 +7980,38 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { } static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7924,14 +8027,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7964,7 +8067,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7987,15 +8091,36 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +static NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; @@ -8535,8 +8660,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index bd868b87a407d67..76be2c7c5ca1bab 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -169,8 +169,8 @@ typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding {

1 0

[3.15] gh-156707: Do not follow junctions in os_helper.rmtree() on Windows (GH-156710) (#156714)
by hugovk Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/b93576a2b66d98ec8fe2bf64de6fd27a78… commit: b93576a2b66d98ec8fe2bf64de6fd27a7851e1ed branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2026-08-31T22:05:08+03:00 summary: [3.15] gh-156707: Do not follow junctions in os_helper.rmtree() on Windows (GH-156710) (#156714) gh-156707: Do not follow junctions in os_helper.rmtree() on Windows (GH-156710) os.lstat() reports a junction as a directory, so the junction was followed and files in the directory it points to could be removed. Now the junction itself is removed, as in os.walk() and shutil.rmtree(). (cherry picked from commit d87ee279a1f6dfdb478ede5f0ba0360123d85916) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: M Lib/test/support/os_helper.py diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index e1e2e69cb3d8334..12d34dedfe1bbaa 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -434,7 +434,10 @@ def _rmtree_inner(path): file=sys.__stderr__) mode = 0 if stat.S_ISDIR(mode): - _waitfor(_rmtree_inner, fullname, waitall=True) + # Do not follow junctions, which os.lstat() reports + # as directories. + if not os.path.isjunction(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) _force_run(fullname, os.rmdir, fullname) else: _force_run(fullname, os.unlink, fullname)

1 0

[3.15] gh-156002: Bound zipfile decompression for bzip2/LZMA/Zstandard (GH-156003) (#156362)
by hugovk Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/1b424c0178a01e155fd0267dc28a8fc115… commit: 1b424c0178a01e155fd0267dc28a8fc1159b33a8 branch: 3.15 author: Petr Viktorin <encukou(a)gmail.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2026-08-31T22:04:04+03:00 summary: [3.15] gh-156002: Bound zipfile decompression for bzip2/LZMA/Zstandard (GH-156003) (#156362) Patch by @tonghuaroot. zipfile.ZipExtFile._read1() bounds the output of each decompress() call for DEFLATE members by passing a max_length to zlib, but for bzip2, LZMA, and Zstandard members it called decompress() with no bound. A whole compressed chunk was therefore expanded into a single allocation before the data[:self._left] clip ran, so a consumer that deliberately reads in small chunks to limit memory (for example zf.open(name).read(8192)) was silently unprotected for non-DEFLATE members. A small, spec-conformant archive member declaring a large uncompressed size could drive multi-GB peak memory. _read1() now passes a per-call bound to the non-DEFLATE decompress() (mirroring the DEFLATE branch) and drains the decompressor's internal buffer across calls by checking needs_input before reading more compressed input. zipfile's LZMADecompressor wrapper forwards max_length and exposes needs_input so the bound also holds for LZMA members. (cherry picked from commit f897dbf2f36a5935700b7c2d94d4681d2136b7d4) Co-authored-by: tonghuaroot <tonghuaroot(a)gmail.com> files: A Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst M Lib/test/test_zipfile/test_core.py M Lib/zipfile/__init__.py diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index feec688726a8038..0abdf87bb5ab2c7 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2719,6 +2719,48 @@ def tearDown(self): unlink(TESTFN2) +class AbstractBoundedDecompressTests: + # ZipExtFile._read1() bounds the output of each decompress() call so that a + # small member declaring a large uncompressed size cannot expand into one + # unbounded read. + def test_read1_output_is_bounded(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: + zf.writestr("big", b"\0" * (4 * 1024 * 1024)) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + with zf.open("big") as f: + self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) + + +class StoredBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_STORED + + +@requires_zlib() +class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + + +@requires_bz2() +class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + + +@requires_lzma() +class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +@requires_zstd() +class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_ZSTANDARD + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 71e4dd4f6f625ce..5a53feed4e3121e 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -786,7 +786,16 @@ def __init__(self): self._unconsumed = b'' self.eof = False - def decompress(self, data): + @property + def _needs_input(self): + # While the LZMA properties header is still being buffered, more input + # is required; afterwards defer to the wrapped decompressor so a bounded + # decompress() call can be drained across reads. + if self._decomp is None: + return True + return self._decomp.needs_input + + def decompress(self, data, max_length=-1): if self._decomp is None: self._unconsumed += data if len(self._unconsumed) <= 4: @@ -802,7 +811,7 @@ def decompress(self, data): data = self._unconsumed[4 + psize:] del self._unconsumed - result = self._decomp.decompress(data) + result = self._decomp.decompress(data, max_length) self.eof = self._decomp.eof return result @@ -869,6 +878,13 @@ def _get_compressor(compress_type, compresslevel=None): return None +def _decompressor_needs_input(decompressor): + # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA + # wrapper keeps it private (_needs_input) to avoid adding public API. + needs_input = getattr(decompressor, "needs_input", None) + return decompressor._needs_input if needs_input is None else needs_input + + def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1171,8 +1187,15 @@ def _read1(self, n): data = self._decompressor.unconsumed_tail if n > len(data): data += self._read2(n - len(data)) - else: + elif self._compress_type == ZIP_STORED: data = self._read2(n) + else: + # bzip2/lzma/zstd: a bounded decompress() call may leave input + # buffered inside the decompressor; drain that before reading more. + if _decompressor_needs_input(self._decompressor): + data = self._read2(n) + else: + data = b'' if self._compress_type == ZIP_STORED: self._eof = self._compress_left <= 0 @@ -1185,8 +1208,13 @@ def _read1(self, n): if self._eof: data += self._decompressor.flush() else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 + # Bound the output of a single decompress() call (mirroring the + # DEFLATE path above) so that a small compressed member cannot + # expand into one unbounded read. + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + _decompressor_needs_input(self._decompressor)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst new file mode 100644 index 000000000000000..4e49ad5ce8fa00a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst @@ -0,0 +1,4 @@ +Bound the amount of data :mod:`zipfile` decompresses per read for members +compressed with bzip2, LZMA, or Zstandard, matching the existing limit for +deflate. A small archive member could previously expand into an unbounded +allocation even when read in small chunks.

1 0

[3.15] gh-155999: `tarfile`: handle a member that leaves the destination but comes back (GH-156000) (#156040)
by hugovk Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/038a0915fdbb872f8064c37e67e527a8a7… commit: 038a0915fdbb872f8064c37e67e527a8a7c5236c branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2026-08-31T22:01:32+03:00 summary: [3.15] gh-155999: `tarfile`: handle a member that leaves the destination but comes back (GH-156000) (#156040) gh-155999: `tarfile`: handle a member that leaves the destination but comes back (GH-156000) (cherry picked from commit 97688346ada2df3e5b9c279348862c3d64ab0823) Co-authored-by: Stan Ulbrych <stan(a)python.org> files: A Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 29a329fdfeab15..dbd5c418bc2e25 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1107,6 +1107,10 @@ reused in custom filters: paths (in case the name is absolute even after stripping slashes, e.g. ``C:/foo`` on Windows). This raises :class:`~tarfile.AbsolutePathError`. + - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components + using :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the name if it traverses symbolic links. - :ref:`Refuse <tarfile-extraction-refuse>` to extract files whose absolute path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. @@ -1115,6 +1119,10 @@ reused in custom filters: Return the modified ``TarInfo`` member. + .. versionchanged:: next + + Filenames containing ``..`` components are now normalized. + .. function:: data_filter(member, path) Implements the ``'data'`` filter. diff --git a/Lib/tarfile.py b/Lib/tarfile.py index fe28ea68cfd132..59543072e6d271 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -833,6 +833,13 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination + if '..' in name.replace(os.sep, '/').split('/'): + # Directories are created from the name as given, so a name that + # leaves the destination part-way through would create them + # outside it even if the resolved path stays inside. + normalized = os.path.normpath(name) + if normalized != name: + name = new_attrs['name'] = normalized target_path = os.path.realpath(os.path.join(dest_path, name), strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2c6f0da7887f53..6945a9e735bd18 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3968,6 +3968,20 @@ def test_absolute(self): tarfile.AbsolutePathError, """['"].*escaped.evil['"] has an absolute path""") + def test_parent_dir_out_and_back(self): + # Test a member that leaves the destination and comes back. + # The containment check looks at the resolved path, which stays + # inside, but the intermediate directories are created from the + # name as given, which does not. + with ArchiveMaker() as arc: + arc.add(f'../escaped.evil/../{self.destdir.name}/sub/file', + content='content') + + for filter in 'tar', 'data': + with self.subTest(filter): + with self.check_context(arc.open(), filter): + self.expect_file('sub/file', content='content') + @symlink_test def test_parent_symlink(self): # Test interplaying symlinks diff --git a/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst new file mode 100644 index 00000000000000..59b725e55bbffd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-13-13-08-11.gh-issue-155999.Xt4rWq.rst @@ -0,0 +1,5 @@ +Fix the :mod:`tarfile` ``tar`` and ``data`` extraction filters creating +directories outside the destination for members whose name leaves the +destination and returns to it, such as ``../evil/../dest/sub/file``. The +containment check used the resolved path, but intermediate directories were +created from the name as given.

1 0

[3.15] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156731)
by hugovk Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/3122687aa4093f9cdf0828ea45f4b80567… commit: 3122687aa4093f9cdf0828ea45f4b805670e7d65 branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2026-08-31T18:49:00Z summary: [3.15] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156731) gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (cherry picked from commit 287b7cffb79d443614be51b48eaf085d663aeecd) Co-authored-by: Stan Ulbrych <stan(a)python.org> files: A Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst D Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst M Misc/sbom.spdx.json M Modules/expat/expat.h M Modules/expat/internal.h M Modules/expat/refresh.sh M Modules/expat/xmlparse.c M Modules/expat/xmltok.h diff --git a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst similarity index 59% rename from Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst rename to Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst index 439366c8633e824..3dda6055f307d94 100644 --- a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst +++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst @@ -1,2 +1 @@ -Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.3 -for the fix to :cve:`2026-72522`. +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.4. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 9753f23ff8ed260..86811220b4124a4 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7baecf6e04769cfb0c5ce2a6e3241e3a0bb8c9e9" + "checksumValue": "12dffaa4a67cbe308643dbec7ffc1b4fd38abbde" }, { "algorithm": "SHA256", - "checksumValue": "d3f19ed52dc975741ecc5a0fc553f910a241d60c76fa4621356d0cdb0490ca28" + "checksumValue": "0e912e25375e213b6e4ff90d554e0a0e037e6f0c20dfa734d366e5bdff289f20" } ], "fileName": "Modules/expat/expat.h" @@ -104,11 +104,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "476a11d9872f8f38844e398c5486ad183ffe2dcf" + "checksumValue": "4afd563c90edd6b4aa5abedcd3df5df023668d26" }, { "algorithm": "SHA256", - "checksumValue": "89f661fa3fa5f7892d83a13ecd685a56aace3fe740abce88a863031114ee2cef" + "checksumValue": "beb7211c800d827743bd3d6ddb86538302d6c51180be6d3b61a1c315e061762d" } ], "fileName": "Modules/expat/internal.h" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0939e3fe0ebb21a5b8ed9d9fdd33cde75ee5658a" + "checksumValue": "b9e4628f37353a7eec8a98c26ebb88d7e2d48971" }, { "algorithm": "SHA256", - "checksumValue": "da48375e85bdc2f97da4445169aafc0b363f150a1a8275dd417e6d84cfc3e443" + "checksumValue": "9afa5cb812283750f1970e230ba392201bac46240abf51ab65e5090e93ca34a6" } ], "fileName": "Modules/expat/xmlparse.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e4bf167669dddff38269486f33eccb0fde0c7ca" + "checksumValue": "e9a5972f664c1c530443ddd8b7900e406e3ca02a" }, { "algorithm": "SHA256", - "checksumValue": "20013b75027e04e324452a002100076e30ec20e0f28b318f392317f99a4c4115" + "checksumValue": "41a6cef659ef1da9ee732304332c4134afe4b63571442de77eaf9918abf9e5df" } ], "fileName": "Modules/expat/xmltok.h" @@ -1772,14 +1772,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" + "checksumValue": "b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.…", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.…", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1787,7 +1787,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.3" + "versionInfo": "2.8.4" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index dbebd985a652ac7..b296be9dbad2dc6 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1096,7 +1096,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 3 +# define XML_MICRO_VERSION 4 # ifdef __cplusplus } diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 7e67d2e378c5243..6311028e94b8f5f 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -33,6 +33,7 @@ Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild(a)sony.com> Copyright (c) 2024 Taichi Haradaguchi <20001722(a)ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka <mattheww(a)simba.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 1499e92112fb952..ef06e87122aa260 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_3" -expected_libexpat_version="2.8.3" -expected_libexpat_sha256="22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" +expected_libexpat_tag="R_2_8_4" +expected_libexpat_version="2.8.4" +expected_libexpat_sha256="b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 4fa61bca8c16293..9a05da21d5a7fd3 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ee5f82c3ffd57c5224394ba46f348dbce466d34d6c925a527ae46b1cfe6adf1d (2.8.3+) +/* 13c4e8da8fccffb0e8e599684e0d447ad14c1bb0b48792cf5dd77d8712301871 (2.8.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -51,6 +51,9 @@ Copyright (c) 2026 Kartik Kenchi <netliomax25(a)gmail.com> Copyright (c) 2026 Haris Hussain <hextheshadow0x(a)gmail.com> Copyright (c) 2026 Evgeny Kotkov <kotkov(a)apache.org> + Copyright (c) 2026 Darren Carreras <carrerasdarren(a)gmail.com> + Copyright (c) 2026 Alberto Maschietto <albertomaschietto9(a)gmail.com> + Copyright (c) 2026 Zeyou Liu <zeyouliu(a)tencent.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -330,7 +333,7 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; + bool open; XML_Bool hasMore; /* true if entity has not been completely processed */ /* An entity can be open while being already completely processed (hasMore == XML_FALSE). The reason is the delayed closing of entities until their inner @@ -381,6 +384,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -394,7 +413,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -579,6 +598,8 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); +static NAMED *lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, + size_t nameLen, size_t createSize); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); @@ -755,6 +776,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -1177,6 +1200,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1524,6 +1566,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1604,7 +1647,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); @@ -1915,7 +1958,7 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); FREE(parser, parser); } @@ -2739,7 +2782,7 @@ XML_GetCurrentLineNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -2756,7 +2799,7 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -3410,9 +3453,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3837,8 +3880,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3951,11 +3994,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4046,8 +4092,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4946,25 +4992,34 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } @@ -6092,7 +6147,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6101,11 +6156,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6429,7 +6484,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; entity->hasMore = XML_TRUE; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); @@ -6520,7 +6575,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6598,7 +6653,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6894,7 +6949,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (entity->systemId) { if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6903,12 +6958,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7058,7 +7113,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7239,7 +7294,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7275,11 +7330,24 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7480,7 +7548,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7597,7 +7665,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); @@ -7639,7 +7707,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7732,8 +7800,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7766,11 +7834,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7867,19 +7946,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } @@ -7897,18 +7980,38 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { } static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7924,14 +8027,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7964,7 +8067,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7987,15 +8091,36 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +static NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; @@ -8535,8 +8660,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index bd868b87a407d67..76be2c7c5ca1bab 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -169,8 +169,8 @@ typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding {

1 0

[3.13] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156733)
by StanFromIreland Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/a371ba8c77d2913dd66477840b145e8984… commit: a371ba8c77d2913dd66477840b145e898461e263 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: StanFromIreland <stan(a)python.org> date: 2026-08-31T18:44:43Z summary: [3.13] gh-156723: Update bundled libexpat to version 2.8.4 (GH-156724) (#156733) (cherry picked from commit 287b7cffb79d443614be51b48eaf085d663aeecd) Co-authored-by: Stan Ulbrych <stan(a)python.org> files: A Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst D Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst M Misc/sbom.spdx.json M Modules/expat/expat.h M Modules/expat/internal.h M Modules/expat/refresh.sh M Modules/expat/xmlparse.c M Modules/expat/xmltok.h diff --git a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst similarity index 59% rename from Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst rename to Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst index 439366c8633e824..3dda6055f307d94 100644 --- a/Misc/NEWS.d/next/Security/2026-08-11-15-24-21.gh-issue-155558.yDsXZC.rst +++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-156723.KxCF5N.rst @@ -1,2 +1 @@ -Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.3 -for the fix to :cve:`2026-72522`. +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.4. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index b801a6386e56166..93fbe6faba4b9e8 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7baecf6e04769cfb0c5ce2a6e3241e3a0bb8c9e9" + "checksumValue": "12dffaa4a67cbe308643dbec7ffc1b4fd38abbde" }, { "algorithm": "SHA256", - "checksumValue": "d3f19ed52dc975741ecc5a0fc553f910a241d60c76fa4621356d0cdb0490ca28" + "checksumValue": "0e912e25375e213b6e4ff90d554e0a0e037e6f0c20dfa734d366e5bdff289f20" } ], "fileName": "Modules/expat/expat.h" @@ -104,11 +104,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "476a11d9872f8f38844e398c5486ad183ffe2dcf" + "checksumValue": "4afd563c90edd6b4aa5abedcd3df5df023668d26" }, { "algorithm": "SHA256", - "checksumValue": "89f661fa3fa5f7892d83a13ecd685a56aace3fe740abce88a863031114ee2cef" + "checksumValue": "beb7211c800d827743bd3d6ddb86538302d6c51180be6d3b61a1c315e061762d" } ], "fileName": "Modules/expat/internal.h" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0939e3fe0ebb21a5b8ed9d9fdd33cde75ee5658a" + "checksumValue": "b9e4628f37353a7eec8a98c26ebb88d7e2d48971" }, { "algorithm": "SHA256", - "checksumValue": "da48375e85bdc2f97da4445169aafc0b363f150a1a8275dd417e6d84cfc3e443" + "checksumValue": "9afa5cb812283750f1970e230ba392201bac46240abf51ab65e5090e93ca34a6" } ], "fileName": "Modules/expat/xmlparse.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8e4bf167669dddff38269486f33eccb0fde0c7ca" + "checksumValue": "e9a5972f664c1c530443ddd8b7900e406e3ca02a" }, { "algorithm": "SHA256", - "checksumValue": "20013b75027e04e324452a002100076e30ec20e0f28b318f392317f99a4c4115" + "checksumValue": "41a6cef659ef1da9ee732304332c4134afe4b63571442de77eaf9918abf9e5df" } ], "fileName": "Modules/expat/xmltok.h" @@ -1590,14 +1590,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" + "checksumValue": "b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.…", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.…", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1605,7 +1605,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.3" + "versionInfo": "2.8.4" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index dbebd985a652ac7..b296be9dbad2dc6 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1096,7 +1096,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 3 +# define XML_MICRO_VERSION 4 # ifdef __cplusplus } diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 7e67d2e378c5243..6311028e94b8f5f 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -33,6 +33,7 @@ Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild(a)sony.com> Copyright (c) 2024 Taichi Haradaguchi <20001722(a)ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka <mattheww(a)simba.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index f30c01f83bcc4bc..641b2ac910fd757 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_3" -expected_libexpat_version="2.8.3" -expected_libexpat_sha256="22920a86c83f32300b11463635b71f11137a917975af297725e55525027d4e50" +expected_libexpat_tag="R_2_8_4" +expected_libexpat_version="2.8.4" +expected_libexpat_sha256="b8ece2437692dad44d851c4532723390a5a330990007706be9c8d2b90d294f36" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 4fa61bca8c16293..9a05da21d5a7fd3 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* ee5f82c3ffd57c5224394ba46f348dbce466d34d6c925a527ae46b1cfe6adf1d (2.8.3+) +/* 13c4e8da8fccffb0e8e599684e0d447ad14c1bb0b48792cf5dd77d8712301871 (2.8.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -51,6 +51,9 @@ Copyright (c) 2026 Kartik Kenchi <netliomax25(a)gmail.com> Copyright (c) 2026 Haris Hussain <hextheshadow0x(a)gmail.com> Copyright (c) 2026 Evgeny Kotkov <kotkov(a)apache.org> + Copyright (c) 2026 Darren Carreras <carrerasdarren(a)gmail.com> + Copyright (c) 2026 Alberto Maschietto <albertomaschietto9(a)gmail.com> + Copyright (c) 2026 Zeyou Liu <zeyouliu(a)tencent.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -330,7 +333,7 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; + bool open; XML_Bool hasMore; /* true if entity has not been completely processed */ /* An entity can be open while being already completely processed (hasMore == XML_FALSE). The reason is the delayed closing of entities until their inner @@ -381,6 +384,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -394,7 +413,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -579,6 +598,8 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); +static NAMED *lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, + size_t nameLen, size_t createSize); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); @@ -755,6 +776,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -1177,6 +1200,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1524,6 +1566,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1604,7 +1647,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); @@ -1915,7 +1958,7 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + callUnknownEncodingRelease(parser); FREE(parser, parser); } @@ -2739,7 +2782,7 @@ XML_GetCurrentLineNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -2756,7 +2799,7 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } - // NOTE: XML_Size is known to wrap around for >2 4iB content + // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break @@ -3410,9 +3453,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3837,8 +3880,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3951,11 +3994,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4046,8 +4092,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4946,25 +4992,34 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } @@ -6092,7 +6147,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6101,11 +6156,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6429,7 +6484,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; entity->hasMore = XML_TRUE; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); @@ -6520,7 +6575,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6598,7 +6653,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6894,7 +6949,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (entity->systemId) { if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6903,12 +6958,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7058,7 +7113,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7239,7 +7294,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7275,11 +7330,24 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7480,7 +7548,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7597,7 +7665,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); @@ -7639,7 +7707,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7732,8 +7800,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7766,11 +7834,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7867,19 +7946,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } @@ -7897,18 +7980,38 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { } static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7924,14 +8027,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7964,7 +8067,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7987,15 +8091,36 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +static NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; @@ -8535,8 +8660,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index bd868b87a407d67..76be2c7c5ca1bab 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -169,8 +169,8 @@ typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding {

1 0

gh-121291: Respect mixin bitwise-operator overrides on Flag subclasses (GH-155862)
by ethanfurman Aug. 31, 2026

Aug. 31, 2026

https://github.com/python/cpython/commit/486b000c6c19c555f03b481f735f4dec49… commit: 486b000c6c19c555f03b481f735f4dec498f0f67 branch: main author: Som Samantray <92726151+SomSamantray(a)users.noreply.github.com> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2026-08-31T11:39:49-07:00 summary: gh-121291: Respect mixin bitwise-operator overrides on Flag subclasses (GH-155862) * gh-121291: respect mixin-defined bitwise operators on Flag subclasses EnumMeta.__new__ unconditionally installed Flag's __or__/__and__/__xor__/ __ror__/__rand__/__rxor__/__invert__ onto every Flag subclass, silently discarding a mixin base's own override of these dunders -- even though the class's MRO should have resolved to the mixin's method; this fixes that. files: A Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst M Lib/enum.py M Lib/test/test_enum.py diff --git a/Lib/enum.py b/Lib/enum.py index 7aff36c94ce1dc..076aa18a02fd20 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -624,9 +624,13 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k '__invert__' ): if name not in classdict: + # check for mixin overrides before replacing enum_method = getattr(Flag, name) - setattr(enum_class, name, enum_method) - classdict[name] = enum_method + found_method = getattr(enum_class, name) + data_type_method = getattr(member_type, name, None) + if found_method in (enum_method, data_type_method): + setattr(enum_class, name, enum_method) + classdict[name] = enum_method # # replace any other __new__ with our own (as long as Enum is not None, # anyway) -- again, this is to support pickle diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index b05eab43bd9eff..447f847f33da93 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4080,6 +4080,39 @@ class NeverEnum(WhereEnum): self.assertFalse(NeverEnum.__dict__.get('_test1', False)) self.assertFalse(NeverEnum.__dict__.get('_test2', False)) + def test_mixin_operator_override(self): + # a mixin's own bitwise-operator overrides must not be clobbered + # by Flag's default __or__/__and__/__xor__/__invert__ -- gh-121291 + class OperatorMixin: + def __or__(self, other): + return 'mixin-or' + def __ror__(self, other): + return 'mixin-ror' + def __invert__(self): + return 'mixin-invert' + class MixedFlag(OperatorMixin, Flag): + A = 1 + B = 2 + self.assertIs(MixedFlag.__or__, OperatorMixin.__or__) + self.assertIs(MixedFlag.__ror__, OperatorMixin.__ror__) + self.assertIs(MixedFlag.__invert__, OperatorMixin.__invert__) + self.assertEqual(MixedFlag.A | MixedFlag.B, 'mixin-or') + self.assertEqual(1 | MixedFlag.A, 'mixin-ror') + self.assertEqual(~MixedFlag.A, 'mixin-invert') + # dunders the mixin didn't override still get Flag's own + self.assertIs(MixedFlag.__and__, Flag.__and__) + self.assertIs(MixedFlag.__xor__, Flag.__xor__) + self.assertIs(MixedFlag.__rand__, Flag.__rand__) + self.assertIs(MixedFlag.__rxor__, Flag.__rxor__) + self.assertEqual(MixedFlag.A & MixedFlag.B, MixedFlag(0)) + # + # a plain (non-mixin) Flag subclass is unaffected + class PlainFlag(Flag): + A = 1 + B = 2 + self.assertIs(PlainFlag.__or__, Flag.__or__) + self.assertEqual(PlainFlag.A | PlainFlag.B, PlainFlag(3)) + class OldTestIntFlag(unittest.TestCase): """Tests of the IntFlags.""" @@ -4564,6 +4597,26 @@ def cycle_enum(): 'at least one thread failed while creating composite members') self.assertEqual(256, len(seen), 'too many composite members created') + def test_mixin_operator_override(self): + # IntFlag's own mixed-in `int` also defines these operators, so the + # fix for gh-121291 must still override `int`'s raw operators with + # Flag's (returning IntFlag instances, not plain ints), while still + # respecting a genuine, separate mixin's override. + Color = self.Color + combined = Color.RED | Color.BLUE + self.assertIs(type(combined), Color) + self.assertEqual(combined, Color.PURPLE) + self.assertEqual(repr(combined), '<Color.PURPLE: 5>') + # + class OperatorMixin: + def __or__(self, other): + return 'mixin-or' + class MixedIntFlag(OperatorMixin, IntFlag): + A = 1 + B = 2 + self.assertIs(MixedIntFlag.__or__, OperatorMixin.__or__) + self.assertEqual(MixedIntFlag.A | MixedIntFlag.B, 'mixin-or') + class TestEmptyAndNonLatinStrings(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst b/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst new file mode 100644 index 00000000000000..e6d9f4b69ec9e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-15-18-16-16.gh-issue-121291.WljPkh.rst @@ -0,0 +1,4 @@ +:class:`enum.Flag` (and :class:`enum.IntFlag`) subclasses no longer have +a mixin base's own ``__or__``, ``__and__``, ``__xor__``, ``__ror__``, +``__rand__``, ``__rxor__``, or ``__invert__`` override silently replaced +by :class:`~enum.Flag`'s default implementation.

1 0