Mailman 3 August 2022 - Python-checkins

[3.11] gh-96189: Fix test_invalid_utf8 on a number of build bots (GH-96190) (#96218)
by pablogsal Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/c01fc9d30a2a23c8f9f2bcdd85e0c87acf… commit: c01fc9d30a2a23c8f9f2bcdd85e0c87acfc57e72 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2022-08-23T23:28:44+01:00 summary: [3.11] gh-96189: Fix test_invalid_utf8 on a number of build bots (GH-96190) (#96218) Co-authored-by: Michael Droettboom <mdboom(a)gmail.com> files: M Lib/test/test_source_encoding.py diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index e1b0de2adef..8d7b573c7e9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -239,50 +239,50 @@ def test_invalid_utf8(self): # it's an otherwise valid Python source file. template = b'"%s"\n' - with tempfile.TemporaryDirectory() as tmpd: - fn = os.path.join(tmpd, 'test.py') + fn = TESTFN + self.addCleanup(unlink, fn) + + def check(content): + with open(fn, 'wb') as fp: + fp.write(template % content) + script_helper.assert_python_failure(fn) + + # continuation bytes in a sequence of 2, 3, or 4 bytes + continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] + # start bytes of a 2-byte sequence equivalent to code points < 0x7F + invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] + # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF + invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] + invalid_start_bytes = ( + continuation_bytes + invalid_2B_seq_start_bytes + + invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] + ) - def check(content): - with open(fn, 'wb') as fp: - fp.write(template % content) - script_helper.assert_python_failure(fn) - - # continuation bytes in a sequence of 2, 3, or 4 bytes - continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] - # start bytes of a 2-byte sequence equivalent to code points < 0x7F - invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] - # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF - invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] - invalid_start_bytes = ( - continuation_bytes + invalid_2B_seq_start_bytes + - invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] - ) - - for byte in invalid_start_bytes: - check(byte) - - for sb in invalid_2B_seq_start_bytes: - for cb in continuation_bytes: - check(sb + cb) - - for sb in invalid_4B_seq_start_bytes: - for cb1 in continuation_bytes[:3]: - for cb3 in continuation_bytes[:3]: - check(sb+cb1+b'\x80'+cb3) - - for cb in [bytes([x]) for x in range(0x80, 0xA0)]: - check(b'\xE0'+cb+b'\x80') - check(b'\xE0'+cb+b'\xBF') - # surrogates - for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: - check(b'\xED'+cb+b'\x80') - check(b'\xED'+cb+b'\xBF') - for cb in [bytes([x]) for x in range(0x80, 0x90)]: - check(b'\xF0'+cb+b'\x80\x80') - check(b'\xF0'+cb+b'\xBF\xBF') - for cb in [bytes([x]) for x in range(0x90, 0xC0)]: - check(b'\xF4'+cb+b'\x80\x80') - check(b'\xF4'+cb+b'\xBF\xBF') + for byte in invalid_start_bytes: + check(byte) + + for sb in invalid_2B_seq_start_bytes: + for cb in continuation_bytes: + check(sb + cb) + + for sb in invalid_4B_seq_start_bytes: + for cb1 in continuation_bytes[:3]: + for cb3 in continuation_bytes[:3]: + check(sb+cb1+b'\x80'+cb3) + + for cb in [bytes([x]) for x in range(0x80, 0xA0)]: + check(b'\xE0'+cb+b'\x80') + check(b'\xE0'+cb+b'\xBF') + # surrogates + for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: + check(b'\xED'+cb+b'\x80') + check(b'\xED'+cb+b'\xBF') + for cb in [bytes([x]) for x in range(0x80, 0x90)]: + check(b'\xF0'+cb+b'\x80\x80') + check(b'\xF0'+cb+b'\xBF\xBF') + for cb in [bytes([x]) for x in range(0x90, 0xC0)]: + check(b'\xF4'+cb+b'\x80\x80') + check(b'\xF4'+cb+b'\xBF\xBF') class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):

1 0

GH-96145: Add AttrDict to JSON module for use with object_hook (#96146)
by rhettinger Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/1f0eafa844bf5a380603d55e8d4b42d8c2… commit: 1f0eafa844bf5a380603d55e8d4b42d8c2a3439d branch: main author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2022-08-23T16:22:00-05:00 summary: GH-96145: Add AttrDict to JSON module for use with object_hook (#96146) files: A Lib/test/test_json/test_attrdict.py A Misc/NEWS.d/next/Library/2022-08-20-12-56-15.gh-issue-96145.8ah3pE.rst M Doc/library/json.rst M Lib/json/__init__.py M Lib/test/test_json/__init__.py diff --git a/Doc/library/json.rst b/Doc/library/json.rst index f65be85d31bf..467d5d9e1544 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -9,6 +9,11 @@ **Source code:** :source:`Lib/json/__init__.py` +.. testsetup:: * + + import json + from json import AttrDict + -------------- `JSON (JavaScript Object Notation) <https://json.org>`_, specified by @@ -532,6 +537,44 @@ Exceptions .. versionadded:: 3.5 +.. class:: AttrDict(**kwargs) + AttrDict(mapping, **kwargs) + AttrDict(iterable, **kwargs) + + Subclass of :class:`dict` object that also supports attribute style dotted access. + + This class is intended for use with the :attr:`object_hook` in + :func:`json.load` and :func:`json.loads`:: + + .. doctest:: + + >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' + >>> orbital_period = json.loads(json_string, object_hook=AttrDict) + >>> orbital_period['earth'] # Dict style lookup + 365 + >>> orbital_period.earth # Attribute style lookup + 365 + >>> orbital_period.keys() # All dict methods are present + dict_keys(['mercury', 'venus', 'earth', 'mars']) + + Attribute style access only works for keys that are valid attribute + names. In contrast, dictionary style access works for all keys. For + example, ``d.two words`` contains a space and is not syntactically + valid Python, so ``d["two words"]`` should be used instead. + + If a key has the same name as a dictionary method, then a dictionary + lookup finds the key and an attribute lookup finds the method: + + .. doctest:: + + >>> d = AttrDict(items=50) + >>> d['items'] # Lookup the key + 50 + >>> d.items() # Call the method + dict_items([('items', 50)]) + + .. versionadded:: 3.12 + Standard Compliance and Interoperability ---------------------------------------- diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index e4c21daaf3e4..d775fb1c1107 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -97,7 +97,7 @@ """ __version__ = '2.0.9' __all__ = [ - 'dump', 'dumps', 'load', 'loads', + 'dump', 'dumps', 'load', 'loads', 'AttrDict', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', ] @@ -357,3 +357,53 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(**kw).decode(s) + +class AttrDict(dict): + """Dict like object that supports attribute style dotted access. + + This class is intended for use with the *object_hook* in json.loads(): + + >>> from json import loads, AttrDict + >>> json_string = '{"mercury": 88, "venus": 225, "earth": 365, "mars": 687}' + >>> orbital_period = loads(json_string, object_hook=AttrDict) + >>> orbital_period['earth'] # Dict style lookup + 365 + >>> orbital_period.earth # Attribute style lookup + 365 + >>> orbital_period.keys() # All dict methods are present + dict_keys(['mercury', 'venus', 'earth', 'mars']) + + Attribute style access only works for keys that are valid attribute names. + In contrast, dictionary style access works for all keys. + For example, ``d.two words`` contains a space and is not syntactically + valid Python, so ``d["two words"]`` should be used instead. + + If a key has the same name as dictionary method, then a dictionary + lookup finds the key and an attribute lookup finds the method: + + >>> d = AttrDict(items=50) + >>> d['items'] # Lookup the key + 50 + >>> d.items() # Call the method + dict_items([('items', 50)]) + + """ + __slots__ = () + + def __getattr__(self, attr): + try: + return self[attr] + except KeyError: + raise AttributeError(attr) from None + + def __setattr__(self, attr, value): + self[attr] = value + + def __delattr__(self, attr): + try: + del self[attr] + except KeyError: + raise AttributeError(attr) from None + + def __dir__(self): + return list(self) + dir(type(self)) diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index 74b64ed86a31..37b2e0d5e26d 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -18,6 +18,7 @@ class PyTest(unittest.TestCase): json = pyjson loads = staticmethod(pyjson.loads) dumps = staticmethod(pyjson.dumps) + AttrDict = pyjson.AttrDict JSONDecodeError = staticmethod(pyjson.JSONDecodeError) @unittest.skipUnless(cjson, 'requires _json') diff --git a/Lib/test/test_json/test_attrdict.py b/Lib/test/test_json/test_attrdict.py new file mode 100644 index 000000000000..48d14f4db93c --- /dev/null +++ b/Lib/test/test_json/test_attrdict.py @@ -0,0 +1,145 @@ +from test.test_json import PyTest +import pickle +import sys +import unittest + +kepler_dict = { + "orbital_period": { + "mercury": 88, + "venus": 225, + "earth": 365, + "mars": 687, + "jupiter": 4331, + "saturn": 10_756, + "uranus": 30_687, + "neptune": 60_190, + }, + "dist_from_sun": { + "mercury": 58, + "venus": 108, + "earth": 150, + "mars": 228, + "jupiter": 778, + "saturn": 1_400, + "uranus": 2_900, + "neptune": 4_500, + } +} + +class TestAttrDict(PyTest): + + def test_dict_subclass(self): + self.assertTrue(issubclass(self.AttrDict, dict)) + + def test_slots(self): + d = self.AttrDict(x=1, y=2) + with self.assertRaises(TypeError): + vars(d) + + def test_constructor_signatures(self): + AttrDict = self.AttrDict + target = dict(x=1, y=2) + self.assertEqual(AttrDict(x=1, y=2), target) # kwargs + self.assertEqual(AttrDict(dict(x=1, y=2)), target) # mapping + self.assertEqual(AttrDict(dict(x=1, y=0), y=2), target) # mapping, kwargs + self.assertEqual(AttrDict([('x', 1), ('y', 2)]), target) # iterable + self.assertEqual(AttrDict([('x', 1), ('y', 0)], y=2), target) # iterable, kwargs + + def test_getattr(self): + d = self.AttrDict(x=1, y=2) + self.assertEqual(d.x, 1) + with self.assertRaises(AttributeError): + d.z + + def test_setattr(self): + d = self.AttrDict(x=1, y=2) + d.x = 3 + d.z = 5 + self.assertEqual(d, dict(x=3, y=2, z=5)) + + def test_delattr(self): + d = self.AttrDict(x=1, y=2) + del d.x + self.assertEqual(d, dict(y=2)) + with self.assertRaises(AttributeError): + del d.z + + def test_dir(self): + d = self.AttrDict(x=1, y=2) + self.assertTrue(set(dir(d)), set(dir(dict)).union({'x', 'y'})) + + def test_repr(self): + # This repr is doesn't round-trip. It matches a regular dict. + # That seems to be the norm for AttrDict recipes being used + # in the wild. Also it supports the design concept that an + # AttrDict is just like a regular dict but has optional + # attribute style lookup. + self.assertEqual(repr(self.AttrDict(x=1, y=2)), + repr(dict(x=1, y=2))) + + def test_overlapping_keys_and_methods(self): + d = self.AttrDict(items=50) + self.assertEqual(d['items'], 50) + self.assertEqual(d.items(), dict(d).items()) + + def test_invalid_attribute_names(self): + d = self.AttrDict({ + 'control': 'normal case', + 'class': 'keyword', + 'two words': 'contains space', + 'hypen-ate': 'contains a hyphen' + }) + self.assertEqual(d.control, dict(d)['control']) + self.assertEqual(d['class'], dict(d)['class']) + self.assertEqual(d['two words'], dict(d)['two words']) + self.assertEqual(d['hypen-ate'], dict(d)['hypen-ate']) + + def test_object_hook_use_case(self): + AttrDict = self.AttrDict + json_string = self.dumps(kepler_dict) + kepler_ad = self.loads(json_string, object_hook=AttrDict) + + self.assertEqual(kepler_ad, kepler_dict) # Match regular dict + self.assertIsInstance(kepler_ad, AttrDict) # Verify conversion + self.assertIsInstance(kepler_ad.orbital_period, AttrDict) # Nested + + # Exercise dotted lookups + self.assertEqual(kepler_ad.orbital_period, kepler_dict['orbital_period']) + self.assertEqual(kepler_ad.orbital_period.earth, + kepler_dict['orbital_period']['earth']) + self.assertEqual(kepler_ad['orbital_period'].earth, + kepler_dict['orbital_period']['earth']) + + # Dict style error handling and Attribute style error handling + with self.assertRaises(KeyError): + kepler_ad.orbital_period['pluto'] + with self.assertRaises(AttributeError): + kepler_ad.orbital_period.Pluto + + # Order preservation + self.assertEqual(list(kepler_ad.items()), list(kepler_dict.items())) + self.assertEqual(list(kepler_ad.orbital_period.items()), + list(kepler_dict['orbital_period'].items())) + + # Round trip + self.assertEqual(self.dumps(kepler_ad), json_string) + + def test_pickle(self): + AttrDict = self.AttrDict + json_string = self.dumps(kepler_dict) + kepler_ad = self.loads(json_string, object_hook=AttrDict) + + # Pickling requires the cached module to be the real module + cached_module = sys.modules.get('json') + sys.modules['json'] = self.json + try: + for protocol in range(6): + kepler_ad2 = pickle.loads(pickle.dumps(kepler_ad, protocol)) + self.assertEqual(kepler_ad2, kepler_ad) + self.assertEqual(type(kepler_ad2), AttrDict) + finally: + sys.modules['json'] = cached_module + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-08-20-12-56-15.gh-issue-96145.8ah3pE.rst b/Misc/NEWS.d/next/Library/2022-08-20-12-56-15.gh-issue-96145.8ah3pE.rst new file mode 100644 index 000000000000..540ec8b71ebf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-20-12-56-15.gh-issue-96145.8ah3pE.rst @@ -0,0 +1 @@ +Add AttrDict to JSON module for use with object_hook.

1 0

gh-96189: Fix test_invalid_utf8 on a number of build bots (GH-96190)
by miss-islington Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/054328f0dd3e9ee5a8a026dfcfa606baf7… commit: 054328f0dd3e9ee5a8a026dfcfa606baf7e9f052 branch: main author: Michael Droettboom <mdboom(a)gmail.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-23T13:52:55-07:00 summary: gh-96189: Fix test_invalid_utf8 on a number of build bots (GH-96190) The clearing of the temporary directory is not working on some platforms and leaving behind files. This has been updated to use the pattern in test_cmd_line.py [1] using the special TESTFN rather than a test directory. [1] https://github.com/python/cpython/blob/main/Lib/test/test_cmd_line.py#L559 files: M Lib/test/test_source_encoding.py diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index e1b0de2adef..8d7b573c7e9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -239,50 +239,50 @@ def test_invalid_utf8(self): # it's an otherwise valid Python source file. template = b'"%s"\n' - with tempfile.TemporaryDirectory() as tmpd: - fn = os.path.join(tmpd, 'test.py') + fn = TESTFN + self.addCleanup(unlink, fn) + + def check(content): + with open(fn, 'wb') as fp: + fp.write(template % content) + script_helper.assert_python_failure(fn) + + # continuation bytes in a sequence of 2, 3, or 4 bytes + continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] + # start bytes of a 2-byte sequence equivalent to code points < 0x7F + invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] + # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF + invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] + invalid_start_bytes = ( + continuation_bytes + invalid_2B_seq_start_bytes + + invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] + ) - def check(content): - with open(fn, 'wb') as fp: - fp.write(template % content) - script_helper.assert_python_failure(fn) - - # continuation bytes in a sequence of 2, 3, or 4 bytes - continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] - # start bytes of a 2-byte sequence equivalent to code points < 0x7F - invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] - # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF - invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] - invalid_start_bytes = ( - continuation_bytes + invalid_2B_seq_start_bytes + - invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] - ) - - for byte in invalid_start_bytes: - check(byte) - - for sb in invalid_2B_seq_start_bytes: - for cb in continuation_bytes: - check(sb + cb) - - for sb in invalid_4B_seq_start_bytes: - for cb1 in continuation_bytes[:3]: - for cb3 in continuation_bytes[:3]: - check(sb+cb1+b'\x80'+cb3) - - for cb in [bytes([x]) for x in range(0x80, 0xA0)]: - check(b'\xE0'+cb+b'\x80') - check(b'\xE0'+cb+b'\xBF') - # surrogates - for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: - check(b'\xED'+cb+b'\x80') - check(b'\xED'+cb+b'\xBF') - for cb in [bytes([x]) for x in range(0x80, 0x90)]: - check(b'\xF0'+cb+b'\x80\x80') - check(b'\xF0'+cb+b'\xBF\xBF') - for cb in [bytes([x]) for x in range(0x90, 0xC0)]: - check(b'\xF4'+cb+b'\x80\x80') - check(b'\xF4'+cb+b'\xBF\xBF') + for byte in invalid_start_bytes: + check(byte) + + for sb in invalid_2B_seq_start_bytes: + for cb in continuation_bytes: + check(sb + cb) + + for sb in invalid_4B_seq_start_bytes: + for cb1 in continuation_bytes[:3]: + for cb3 in continuation_bytes[:3]: + check(sb+cb1+b'\x80'+cb3) + + for cb in [bytes([x]) for x in range(0x80, 0xA0)]: + check(b'\xE0'+cb+b'\x80') + check(b'\xE0'+cb+b'\xBF') + # surrogates + for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: + check(b'\xED'+cb+b'\x80') + check(b'\xED'+cb+b'\xBF') + for cb in [bytes([x]) for x in range(0x80, 0x90)]: + check(b'\xF0'+cb+b'\x80\x80') + check(b'\xF0'+cb+b'\xBF\xBF') + for cb in [bytes([x]) for x in range(0x90, 0xC0)]: + check(b'\xF4'+cb+b'\x80\x80') + check(b'\xF4'+cb+b'\xBF\xBF') class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):

1 0

Drop myself from owners of genobject (#96216)
by 1st1 Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/e046cf872e33cc79891d829235633077c3… commit: e046cf872e33cc79891d829235633077c33c9315 branch: main author: Yury Selivanov <yury(a)edgedb.com> committer: 1st1 <yury(a)edgedb.com> date: 2022-08-23T12:21:10-07:00 summary: Drop myself from owners of genobject (#96216) files: M .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cacd925cf4e3..0e38536ba994 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,7 +9,7 @@ # Core **/*context* @1st1 -**/*genobject* @1st1 @markshannon +**/*genobject* @markshannon **/*hamt* @1st1 Objects/set* @rhettinger Objects/dict* @methane @markshannon

1 0

gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (GH-96176)
by miss-islington Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/d7eea0f1ca29b143f070b36d8df8e278dc… commit: d7eea0f1ca29b143f070b36d8df8e278dc302a90 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-23T10:18:19-07:00 summary: gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (GH-96176) X-Ref: https://github.com/python/typeshed/pull/8590GH-discussion_r951473977 Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> (cherry picked from commit 58f6953d6d3fe20d972bfa2f6e982206adcf1353) Co-authored-by: Kevin Kirsche <Kev.Kirsche+GitHub(a)gmail.com> files: A Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst M Lib/test/test_minidom.py M Lib/xml/dom/minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 97620258d82f..ef38c362103f 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -9,7 +9,7 @@ import pyexpat import xml.dom.minidom -from xml.dom.minidom import parse, Node, Document, parseString +from xml.dom.minidom import parse, Attr, Node, Document, parseString from xml.dom.minidom import getDOMImplementation from xml.parsers.expat import ExpatError @@ -77,6 +77,20 @@ def testParseFromTextFile(self): dom.unlink() self.confirm(isinstance(dom, Document)) + def testAttrModeSetsParamsAsAttrs(self): + attr = Attr("qName", "namespaceURI", "localName", "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, "localName") + + def testAttrModeSetsNonOptionalAttrs(self): + attr = Attr("qName", "namespaceURI", None, "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, attr.name) + def testGetElementsByTagName(self): dom = parse(tstfile) self.confirm(dom.getElementsByTagName("LI") == \ diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index d09ef5e7d037..ef8a159833bb 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -358,6 +358,8 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix + if localName is not None: + self._localName = localName self.childNodes = NodeList() # Add the single child node that represents the value of the attr diff --git a/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst new file mode 100644 index 000000000000..c34eff22b3d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst @@ -0,0 +1 @@ +Fix unused ``localName`` parameter in the ``Attr`` class in :mod:`xml.dom.minidom`.

1 0

gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (GH-96176)
by miss-islington Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/e783a23702f181cdd5b371f180bedbe87a… commit: e783a23702f181cdd5b371f180bedbe87a95920b branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-23T09:42:10-07:00 summary: gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (GH-96176) X-Ref: https://github.com/python/typeshed/pull/8590GH-discussion_r951473977 Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> (cherry picked from commit 58f6953d6d3fe20d972bfa2f6e982206adcf1353) Co-authored-by: Kevin Kirsche <Kev.Kirsche+GitHub(a)gmail.com> files: A Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst M Lib/test/test_minidom.py M Lib/xml/dom/minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 97620258d82f..ef38c362103f 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -9,7 +9,7 @@ import pyexpat import xml.dom.minidom -from xml.dom.minidom import parse, Node, Document, parseString +from xml.dom.minidom import parse, Attr, Node, Document, parseString from xml.dom.minidom import getDOMImplementation from xml.parsers.expat import ExpatError @@ -77,6 +77,20 @@ def testParseFromTextFile(self): dom.unlink() self.confirm(isinstance(dom, Document)) + def testAttrModeSetsParamsAsAttrs(self): + attr = Attr("qName", "namespaceURI", "localName", "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, "localName") + + def testAttrModeSetsNonOptionalAttrs(self): + attr = Attr("qName", "namespaceURI", None, "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, attr.name) + def testGetElementsByTagName(self): dom = parse(tstfile) self.confirm(dom.getElementsByTagName("LI") == \ diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index d09ef5e7d037..ef8a159833bb 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -358,6 +358,8 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix + if localName is not None: + self._localName = localName self.childNodes = NodeList() # Add the single child node that represents the value of the attr diff --git a/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst new file mode 100644 index 000000000000..c34eff22b3d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst @@ -0,0 +1 @@ +Fix unused ``localName`` parameter in the ``Attr`` class in :mod:`xml.dom.minidom`.

1 0

gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (#96176)
by JelleZijlstra Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/58f6953d6d3fe20d972bfa2f6e982206ad… commit: 58f6953d6d3fe20d972bfa2f6e982206adcf1353 branch: main author: Kevin Kirsche <Kev.Kirsche+GitHub(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2022-08-23T09:16:02-07:00 summary: gh-96175: add missing self._localName assignment in `xml.dom.minidom.Attr` (#96176) X-Ref: https://github.com/python/typeshed/pull/8590#discussion_r951473977 Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> files: A Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst M Lib/test/test_minidom.py M Lib/xml/dom/minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 97620258d82f..ef38c362103f 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -9,7 +9,7 @@ import pyexpat import xml.dom.minidom -from xml.dom.minidom import parse, Node, Document, parseString +from xml.dom.minidom import parse, Attr, Node, Document, parseString from xml.dom.minidom import getDOMImplementation from xml.parsers.expat import ExpatError @@ -77,6 +77,20 @@ def testParseFromTextFile(self): dom.unlink() self.confirm(isinstance(dom, Document)) + def testAttrModeSetsParamsAsAttrs(self): + attr = Attr("qName", "namespaceURI", "localName", "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, "localName") + + def testAttrModeSetsNonOptionalAttrs(self): + attr = Attr("qName", "namespaceURI", None, "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, attr.name) + def testGetElementsByTagName(self): dom = parse(tstfile) self.confirm(dom.getElementsByTagName("LI") == \ diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index d09ef5e7d037..ef8a159833bb 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -358,6 +358,8 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix + if localName is not None: + self._localName = localName self.childNodes = NodeList() # Add the single child node that represents the value of the attr diff --git a/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst new file mode 100644 index 000000000000..c34eff22b3d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-22-13-54-20.gh-issue-96175.bH7zGU.rst @@ -0,0 +1 @@ +Fix unused ``localName`` parameter in the ``Attr`` class in :mod:`xml.dom.minidom`.

1 0

GH-96071: add regression test for GH-96071 (GH-96137)
by miss-islington Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/0aed1e71f3bd6cf298b08cf82078b013a3… commit: 0aed1e71f3bd6cf298b08cf82078b013a32362c2 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-23T08:37:19-07:00 summary: GH-96071: add regression test for GH-96071 (GH-96137) Automerge-Triggered-By: GH:ericsnowcurrently (cherry picked from commit 079baee1962ff7c1f4b60f4dd4c803535ecbd18e) Co-authored-by: Kumar Aditya <59607654+kumaraditya303(a)users.noreply.github.com> files: M Lib/test/test_capi.py diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 40750cc49324..e157d9fdc850 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -915,6 +915,21 @@ def callback(): t.start() t.join() + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_gilstate_ensure_no_deadlock(self): + # See https://github.com/python/cpython/issues/96071 + code = textwrap.dedent(f""" + import _testcapi + + def callback(): + print('callback called') + + _testcapi._test_thread_state(callback) + """) + ret = assert_python_ok('-X', 'tracemalloc', '-c', code) + self.assertIn(b'callback called', ret.out) + class Test_testcapi(unittest.TestCase): locals().update((name, getattr(_testcapi, name))

1 0

[3.10] GH-96187: Prevent _PyCode_GetExtra to return garbage for negative indexes (GH-96188). (#96210)
by pablogsal Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/a6c3f6d34792c5f9dc296ce8329938b7b2… commit: a6c3f6d34792c5f9dc296ce8329938b7b2b02ea9 branch: 3.10 author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2022-08-23T15:58:25+01:00 summary: [3.10] GH-96187: Prevent _PyCode_GetExtra to return garbage for negative indexes (GH-96188). (#96210) (cherry picked from commit 16ebae4cd4029205d932751f26c719c6cb8a6e92) Co-authored-by: Pablo Galindo Salgado <Pablogsal(a)gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2022-08-22-21-33-28.gh-issue-96187.W_6SRG.rst M Objects/codeobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-22-21-33-28.gh-issue-96187.W_6SRG.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-22-21-33-28.gh-issue-96187.W_6SRG.rst new file mode 100644 index 000000000000..fd194faa6854 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-08-22-21-33-28.gh-issue-96187.W_6SRG.rst @@ -0,0 +1,2 @@ +Fixed a bug that caused ``_PyCode_GetExtra`` to return garbage for negative +indexes. Patch by Pablo Galindo diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 976cec584e9f..390b76cafd51 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1304,7 +1304,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) PyCodeObject *o = (PyCodeObject*) code; _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) o->co_extra; - if (co_extra == NULL || co_extra->ce_size <= index) { + if (co_extra == NULL || index < 0 || co_extra->ce_size <= index) { *extra = NULL; return 0; }

1 0

GH-96068: Document object layout (GH-96069)
by markshannon Aug. 23, 2022

Aug. 23, 2022

https://github.com/python/cpython/commit/575f8880bf8498ee05a8e197fc2ed85db6… commit: 575f8880bf8498ee05a8e197fc2ed85db6880361 branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2022-08-23T13:55:43+01:00 summary: GH-96068: Document object layout (GH-96069) files: A Objects/object_layout.md A Objects/object_layout_312.gv A Objects/object_layout_312.png A Objects/object_layout_full_312.gv A Objects/object_layout_full_312.png diff --git a/Objects/object_layout.md b/Objects/object_layout.md new file mode 100644 index 00000000000..9380b57938c --- /dev/null +++ b/Objects/object_layout.md @@ -0,0 +1,82 @@ +# Object layout + +## Common header + +Each Python object starts with two fields: + +* ob_refcnt +* ob_type + +which the form the header common to all Python objects, for all versions, +and hold the reference count and class of the object, respectively. + +## Pre-header + +Since the introduction of the cycle GC, there has also been a pre-header. +Before 3.11, this pre-header was two words in size. +It should be considered opaque to all code except the cycle GC. + +## 3.11 pre-header + +In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. +The reason for moving the ``__dict__`` to the pre-header is that it allows +faster access, as it is at a fixed offset, and it also allows object's +dictionaries to be lazily created when the ``__dict__`` attribute is +specifically asked for. + +In the 3.11 the non-GC part of the pre-header consists of two pointers: + +* dict +* values + +The values pointer refers to the ``PyDictValues`` array which holds the +values of the objects's attributes. +Should the dictionary be needed, then ``values`` is set to ``NULL`` +and the ``dict`` field points to the dictionary. + +## 3.12 pre-header + +In 3.12 the the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictioanry, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. + + +## Layout of a "normal" Python object in 3.12: + +* weakreflist +* dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type + +For a "normal" Python object, that is one that doesn't inherit from a builtin +class or have slots, the header and pre-header form the entire object. + +![Layout of "normal" object in 3.12](./object_layout_312.png) + +There are several advantages to this layout: + +* It allows lazy `__dict__`s, as described above. +* The regular layout allows us to create tailored traversal and deallocation + functions based on layout, rather than inheritance. +* Multiple inheritance works properly, + as the weakrefs and dict are always at the same offset. + +The full layout object, with an opaque part defined by a C extension, +and `__slots__` looks like this: + +![Layout of "full" object in 3.12](./object_layout_full_312.png) + diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv new file mode 100644 index 00000000000..c0068d78568 --- /dev/null +++ b/Objects/object_layout_312.gv @@ -0,0 +1,50 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>object</b></td></tr> + <tr><td port="w" border="1">weakrefs</td></tr> + <tr><td port="dv" border="1">dict or values</td></tr> + <tr><td border="1" >GC info 0</td></tr> + <tr><td border="1" >GC info 1</td></tr> + <tr><td port="r" border="1" >refcount</td></tr> + <tr><td port="h" border="1" >__class__</td></tr> + </table>> + ] + + values [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>values</b></td></tr> + <tr><td port="0" border="1">values[0]</td></tr> + <tr><td border="1">values[1]</td></tr> + <tr><td border="1">...</td></tr> + </table>> + + ] + + class [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>class</b></td></tr> + <tr><td port="head" bgcolor="lightgreen" border="1">...</td></tr> + <tr><td border="1" bgcolor="lightgreen">dict_offset</td></tr> + <tr><td border="1" bgcolor="lightgreen">...</td></tr> + <tr><td port="k" border="1" bgcolor="lightgreen">cached_keys</td></tr> + </table>> + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> values:0 + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png new file mode 100644 index 00000000000..396dab183b3 Binary files /dev/null and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_full_312.gv b/Objects/object_layout_full_312.gv new file mode 100644 index 00000000000..522fa32b066 --- /dev/null +++ b/Objects/object_layout_full_312.gv @@ -0,0 +1,25 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>object</b></td></tr> + <tr><td port="w" border="1">weakrefs</td></tr> + <tr><td port="dv" border="1">dict or values</td></tr> + <tr><td border="1" >GC info 0</td></tr> + <tr><td border="1" >GC info 1</td></tr> + <tr><td port="r" border="1" >refcount</td></tr> + <tr><td port="h" border="1" >__class__</td></tr> + <tr><td border="1">opaque (extension) data </td></tr> + <tr><td border="1">...</td></tr> + <tr><td border="1">__slot__ 0</td></tr> + <tr><td border="1">...</td></tr> + </table>> + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_312.png b/Objects/object_layout_full_312.png new file mode 100644 index 00000000000..4f46ca86091 Binary files /dev/null and b/Objects/object_layout_full_312.png differ

1 0