Mailman 3 May 2012 - Python-checkins

cpython: Remove __getslice__, __setslice__, __delslice__ methods from proxies
by richard.oudkerk May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/1f7c452e797e changeset: 77220:1f7c452e797e user: Richard Oudkerk <shibturn(a)gmail.com> date: Tue May 29 12:01:45 2012 +0100 summary: Remove __getslice__, __setslice__, __delslice__ methods from proxies Proxy classes in multiprocessing do not need these methods in Python 3.x. files: Lib/multiprocessing/managers.py | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1035,12 +1035,11 @@ BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__delslice__', - '__getitem__', '__getslice__', '__len__', '__mul__', - '__reversed__', '__rmul__', '__setitem__', '__setslice__', + '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', + '__mul__', '__reversed__', '__rmul__', '__setitem__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' - )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 + )) class ListProxy(BaseListProxy): def __iadd__(self, value): self._callmethod('extend', (value,)) @@ -1058,8 +1057,8 @@ ArrayProxy = MakeProxyType('ArrayProxy', ( - '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__' - )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 + '__len__', '__getitem__', '__setitem__' + )) PoolProxy = MakeProxyType('PoolProxy', ( -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Removed pysetup3.exe executable.
by vinay.sajip May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/01381723bc50 changeset: 77219:01381723bc50 user: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: Tue May 29 11:12:43 2012 +0100 summary: Removed pysetup3.exe executable. files: Lib/venv/scripts/nt/pysetup3.exe | Bin Lib/venv/scripts/nt/pysetup3-script.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/Lib/venv/scripts/nt/pysetup3.exe b/Lib/venv/scripts/nt/pysetup3.exe deleted file mode 100644 Binary file Lib/venv/scripts/nt/pysetup3.exe has changed diff --git a/Lib/venv/scripts/nt/pysetup3-script.py b/Lib/venv/scripts/nt/pysetup3.py rename from Lib/venv/scripts/nt/pysetup3-script.py rename to Lib/venv/scripts/nt/pysetup3.py -- Repository URL: http://hg.python.org/cpython

1 0

cpython: #14835: Make plistlib output empty arrays & dicts like OS X
by hynek.schlawack May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/9e64084f9980 changeset: 77218:9e64084f9980 user: Hynek Schlawack <hs(a)ox.cx> date: Tue May 29 12:04:54 2012 +0200 summary: #14835: Make plistlib output empty arrays & dicts like OS X Patch by Sidney San Martín. files: Lib/plistlib.py | 30 ++++++++++++++++---------- Lib/test/test_plistlib.py | 6 +++++ Misc/ACKS | 1 + Misc/NEWS | 3 ++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/Lib/plistlib.py b/Lib/plistlib.py --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -237,20 +237,26 @@ self.endElement("data") def writeDict(self, d): - self.beginElement("dict") - items = sorted(d.items()) - for key, value in items: - if not isinstance(key, str): - raise TypeError("keys must be strings") - self.simpleElement("key", key) - self.writeValue(value) - self.endElement("dict") + if d: + self.beginElement("dict") + items = sorted(d.items()) + for key, value in items: + if not isinstance(key, str): + raise TypeError("keys must be strings") + self.simpleElement("key", key) + self.writeValue(value) + self.endElement("dict") + else: + self.simpleElement("dict") def writeArray(self, array): - self.beginElement("array") - for value in array: - self.writeValue(value) - self.endElement("array") + if array: + self.beginElement("array") + for value in array: + self.writeValue(value) + self.endElement("array") + else: + self.simpleElement("array") class _InternalDict(dict): diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -55,6 +55,10 @@ </array> <key>aString</key> <string>Doodah</string> + <key>anEmptyDict</key> + <dict/> + <key>anEmptyList</key> + <array/> <key>anInt</key> <integer>728</integer> <key>nestedData</key> @@ -112,6 +116,8 @@ someMoreData = plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10), nestedData = [plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10)], aDate = datetime.datetime(2004, 10, 26, 10, 33, 33), + anEmptyDict = dict(), + anEmptyList = list() ) pl['\xc5benraa'] = "That was a unicode key." return pl diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -662,6 +662,7 @@ Anthony Martin Owen Martin Sébastien Martini +Sidney San Martín Roger Masse Nick Mathewson Simon Mathieu diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #14835: Make plistlib output empty arrays & dicts like OS X. + Patch by Sidney San Martín. + - Issue #14930: Make memoryview objects weakrefable. - Issue #14775: Fix a potential quadratic dict build-up due to the garbage -- Repository URL: http://hg.python.org/cpython

1 0

Daily reference leaks (5c40627e9f11): sum=465
by solipsis＠pitrou.net May 29, 2012

May 29, 2012

results for 5c40627e9f11 on branch "default" -------------------------------------------- test_smtplib leaked [154, 154, 154] references, sum=462 test_super leaked [1, 1, 1] references, sum=3 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog2cWmiD', '-x']

1 0

cpython: Issue 14814: Add namespaces keyword arg to find(*) methods in _elementtree.
by eli.bendersky May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/7d252dbfbee3 changeset: 77217:7d252dbfbee3 user: Eli Bendersky <eliben(a)gmail.com> date: Tue May 29 06:02:56 2012 +0300 summary: Issue 14814: Add namespaces keyword arg to find(*) methods in _elementtree. Add attrib keyword to Element and SubElement in _elementtree. Patch developed with Ezio Melotti. files: Doc/library/xml.etree.elementtree.rst | 29 ++- Lib/test/test_xml_etree.py | 66 ++++++++- Lib/xml/etree/ElementTree.py | 3 + Modules/_elementtree.c | 113 ++++++++++--- 4 files changed, 173 insertions(+), 38 deletions(-) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -476,27 +476,30 @@ .. versionadded:: 3.2 - .. method:: find(match) + .. method:: find(match, namespaces=None) Finds the first subelement matching *match*. *match* may be a tag name or a :ref:`path <elementtree-xpath>`. Returns an element instance - or ``None``. + or ``None``. *namespaces* is an optional mapping from namespace prefix + to full name. - .. method:: findall(match) + .. method:: findall(match, namespaces=None) Finds all matching subelements, by tag name or :ref:`path <elementtree-xpath>`. Returns a list containing all matching - elements in document order. + elements in document order. *namespaces* is an optional mapping from + namespace prefix to full name. - .. method:: findtext(match, default=None) + .. method:: findtext(match, default=None, namespaces=None) Finds text for the first subelement matching *match*. *match* may be a tag name or a :ref:`path <elementtree-xpath>`. Returns the text content of the first matching element, or *default* if no element was found. Note that if the matching element has no text content an empty string - is returned. + is returned. *namespaces* is an optional mapping from namespace prefix + to full name. .. method:: getchildren() @@ -528,11 +531,13 @@ .. versionadded:: 3.2 - .. method:: iterfind(match) + .. method:: iterfind(match, namespaces=None) Finds all matching subelements, by tag name or :ref:`path <elementtree-xpath>`. Returns an iterable yielding all - matching elements in document order. + matching elements in document order. *namespaces* is an optional mapping + from namespace prefix to full name. + .. versionadded:: 3.2 @@ -597,17 +602,17 @@ care. *element* is an element instance. - .. method:: find(match) + .. method:: find(match, namespaces=None) Same as :meth:`Element.find`, starting at the root of the tree. - .. method:: findall(match) + .. method:: findall(match, namespaces=None) Same as :meth:`Element.findall`, starting at the root of the tree. - .. method:: findtext(match, default=None) + .. method:: findtext(match, default=None, namespaces=None) Same as :meth:`Element.findtext`, starting at the root of the tree. @@ -630,7 +635,7 @@ to look for (default is to return all elements) - .. method:: iterfind(match) + .. method:: iterfind(match, namespaces=None) Same as :meth:`Element.iterfind`, starting at the root of the tree. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -62,6 +62,22 @@ </body> """ +SAMPLE_XML_NS_ELEMS = """ +<root> +<h:table xmlns:h="hello"> + <h:tr> + <h:td>Apples</h:td> + <h:td>Bananas</h:td> + </h:tr> +</h:table> + +<f:table xmlns:f="foo"> + <f:name>African Coffee Table</f:name> + <f:width>80</f:width> + <f:length>120</f:length> +</f:table> +</root> +""" def sanity(): """ @@ -1995,6 +2011,17 @@ self.assertEqual(pyET.SubElement.__module__, 'xml.etree.ElementTree') +class NamespaceParseTest(unittest.TestCase): + def test_find_with_namespace(self): + nsmap = {'h': 'hello', 'f': 'foo'} + doc = ET.fromstring(SAMPLE_XML_NS_ELEMS) + + self.assertEqual(len(doc.findall('{hello}table', nsmap)), 1) + self.assertEqual(len(doc.findall('.//{hello}td', nsmap)), 2) + self.assertEqual(len(doc.findall('.//{foo}name', nsmap)), 1) + + + class ElementSlicingTest(unittest.TestCase): def _elem_tags(self, elemlist): return [e.tag for e in elemlist] @@ -2102,6 +2129,41 @@ ERRORS.codes[ERRORS.XML_ERROR_SYNTAX]) +class KeywordArgsTest(unittest.TestCase): + # Test various issues with keyword arguments passed to ET.Element + # constructor and methods + def test_issue14818(self): + x = ET.XML("<a>foo</a>") + self.assertEqual(x.find('a', None), + x.find(path='a', namespaces=None)) + self.assertEqual(x.findtext('a', None, None), + x.findtext(path='a', default=None, namespaces=None)) + self.assertEqual(x.findall('a', None), + x.findall(path='a', namespaces=None)) + self.assertEqual(list(x.iterfind('a', None)), + list(x.iterfind(path='a', namespaces=None))) + + self.assertEqual(ET.Element('a').attrib, {}) + elements = [ + ET.Element('a', dict(href="#", id="foo")), + ET.Element('a', attrib=dict(href="#", id="foo")), + ET.Element('a', dict(href="#"), id="foo"), + ET.Element('a', href="#", id="foo"), + ET.Element('a', dict(href="#", id="foo"), href="#", id="foo"), + ] + for e in elements: + self.assertEqual(e.tag, 'a') + self.assertEqual(e.attrib, dict(href="#", id="foo")) + + e2 = ET.SubElement(elements[0], 'foobar', attrib={'key1': 'value1'}) + self.assertEqual(e2.attrib['key1'], 'value1') + + with self.assertRaisesRegex(TypeError, 'must be dict, not str'): + ET.Element('a', "I'm not a dict") + with self.assertRaisesRegex(TypeError, 'must be dict, not str'): + ET.Element('a', attrib="I'm not a dict") + + # -------------------------------------------------------------------- @@ -2157,7 +2219,9 @@ StringIOTest, ParseErrorTest, ElementTreeTest, - TreeBuilderTest] + NamespaceParseTest, + TreeBuilderTest, + KeywordArgsTest] if module is pyET: # Run the tests specific to the Python implementation test_classes += [NoAcceleratorTest] diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -205,6 +205,9 @@ # constructor def __init__(self, tag, attrib={}, **extra): + if not isinstance(attrib, dict): + raise TypeError("attrib must be dict, not %s" % ( + attrib.__class__.__name__,)) attrib = attrib.copy() attrib.update(extra) self.tag = tag diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -347,6 +347,41 @@ return (PyObject *)e; } +/* Helper function for extracting the attrib dictionary from a keywords dict. + * This is required by some constructors/functions in this module that can + * either accept attrib as a keyword argument or all attributes splashed + * directly into *kwds. + * If there is no 'attrib' keyword, return an empty dict. + */ +static PyObject* +get_attrib_from_keywords(PyObject *kwds) +{ + PyObject *attrib_str = PyUnicode_FromString("attrib"); + PyObject *attrib = PyDict_GetItem(kwds, attrib_str); + + if (attrib) { + /* If attrib was found in kwds, copy its value and remove it from + * kwds + */ + if (!PyDict_Check(attrib)) { + Py_DECREF(attrib_str); + PyErr_Format(PyExc_TypeError, "attrib must be dict, not %.100s", + Py_TYPE(attrib)->tp_name); + return NULL; + } + attrib = PyDict_Copy(attrib); + PyDict_DelItem(kwds, attrib_str); + } else { + attrib = PyDict_New(); + } + + Py_DECREF(attrib_str); + + if (attrib) + PyDict_Update(attrib, kwds); + return attrib; +} + static int element_init(PyObject *self, PyObject *args, PyObject *kwds) { @@ -358,13 +393,23 @@ if (!PyArg_ParseTuple(args, "O|O!:Element", &tag, &PyDict_Type, &attrib)) return -1; - if (attrib || kwds) { - attrib = (attrib) ? PyDict_Copy(attrib) : PyDict_New(); + if (attrib) { + /* attrib passed as positional arg */ + attrib = PyDict_Copy(attrib); if (!attrib) return -1; - if (kwds) - PyDict_Update(attrib, kwds); + if (kwds) { + if (PyDict_Update(attrib, kwds) < 0) { + return -1; + } + } + } else if (kwds) { + /* have keywords args */ + attrib = get_attrib_from_keywords(kwds); + if (!attrib) + return -1; } else { + /* no attrib arg, no kwds, so no attributes */ Py_INCREF(Py_None); attrib = Py_None; } @@ -536,7 +581,7 @@ } static PyObject* -subelement(PyObject* self, PyObject* args, PyObject* kw) +subelement(PyObject *self, PyObject *args, PyObject *kwds) { PyObject* elem; @@ -548,13 +593,23 @@ &PyDict_Type, &attrib)) return NULL; - if (attrib || kw) { - attrib = (attrib) ? PyDict_Copy(attrib) : PyDict_New(); + if (attrib) { + /* attrib passed as positional arg */ + attrib = PyDict_Copy(attrib); if (!attrib) return NULL; - if (kw) - PyDict_Update(attrib, kw); + if (kwds) { + if (PyDict_Update(attrib, kwds) < 0) { + return NULL; + } + } + } else if (kwds) { + /* have keyword args */ + attrib = get_attrib_from_keywords(kwds); + if (!attrib) + return NULL; } else { + /* no attrib arg, no kwds, so no attribute */ Py_INCREF(Py_None); attrib = Py_None; } @@ -881,13 +936,15 @@ } static PyObject* -element_find(ElementObject* self, PyObject* args) +element_find(ElementObject *self, PyObject *args, PyObject *kwds) { int i; PyObject* tag; PyObject* namespaces = Py_None; - - if (!PyArg_ParseTuple(args, "O|O:find", &tag, &namespaces)) + static char *kwlist[] = {"path", "namespaces", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:find", kwlist, + &tag, &namespaces)) return NULL; if (checkpath(tag) || namespaces != Py_None) { @@ -913,15 +970,17 @@ } static PyObject* -element_findtext(ElementObject* self, PyObject* args) +element_findtext(ElementObject *self, PyObject *args, PyObject *kwds) { int i; PyObject* tag; PyObject* default_value = Py_None; PyObject* namespaces = Py_None; _Py_IDENTIFIER(findtext); - - if (!PyArg_ParseTuple(args, "O|OO:findtext", &tag, &default_value, &namespaces)) + static char *kwlist[] = {"path", "default", "namespaces", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:findtext", kwlist, + &tag, &default_value, &namespaces)) return NULL; if (checkpath(tag) || namespaces != Py_None) @@ -951,14 +1010,16 @@ } static PyObject* -element_findall(ElementObject* self, PyObject* args) +element_findall(ElementObject *self, PyObject *args, PyObject *kwds) { int i; PyObject* out; PyObject* tag; PyObject* namespaces = Py_None; - - if (!PyArg_ParseTuple(args, "O|O:findall", &tag, &namespaces)) + static char *kwlist[] = {"path", "namespaces", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:findall", kwlist, + &tag, &namespaces)) return NULL; if (checkpath(tag) || namespaces != Py_None) { @@ -990,13 +1051,15 @@ } static PyObject* -element_iterfind(ElementObject* self, PyObject* args) +element_iterfind(ElementObject *self, PyObject *args, PyObject *kwds) { PyObject* tag; PyObject* namespaces = Py_None; _Py_IDENTIFIER(iterfind); - - if (!PyArg_ParseTuple(args, "O|O:iterfind", &tag, &namespaces)) + static char *kwlist[] = {"path", "namespaces", 0}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:iterfind", kwlist, + &tag, &namespaces)) return NULL; return _PyObject_CallMethodId( @@ -1567,9 +1630,9 @@ {"get", (PyCFunction) element_get, METH_VARARGS}, {"set", (PyCFunction) element_set, METH_VARARGS}, - {"find", (PyCFunction) element_find, METH_VARARGS}, - {"findtext", (PyCFunction) element_findtext, METH_VARARGS}, - {"findall", (PyCFunction) element_findall, METH_VARARGS}, + {"find", (PyCFunction) element_find, METH_VARARGS | METH_KEYWORDS}, + {"findtext", (PyCFunction) element_findtext, METH_VARARGS | METH_KEYWORDS}, + {"findall", (PyCFunction) element_findall, METH_VARARGS | METH_KEYWORDS}, {"append", (PyCFunction) element_append, METH_VARARGS}, {"extend", (PyCFunction) element_extend, METH_VARARGS}, @@ -1578,7 +1641,7 @@ {"iter", (PyCFunction) element_iter, METH_VARARGS}, {"itertext", (PyCFunction) element_itertext, METH_VARARGS}, - {"iterfind", (PyCFunction) element_iterfind, METH_VARARGS}, + {"iterfind", (PyCFunction) element_iterfind, METH_VARARGS | METH_KEYWORDS}, {"getiterator", (PyCFunction) element_iter, METH_VARARGS}, {"getchildren", (PyCFunction) element_getchildren, METH_VARARGS}, -- Repository URL: http://hg.python.org/cpython

2 1

cpython: Refactor test_email/test_pickleable and add tests for date headers
by r.david.murray May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/5c40627e9f11 changeset: 77216:5c40627e9f11 user: R David Murray <rdmurray(a)bitdance.com> date: Mon May 28 21:09:04 2012 -0400 summary: Refactor test_email/test_pickleable and add tests for date headers files: Lib/test/test_email/test_pickleable.py | 87 +++++++++---- 1 files changed, 59 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_email/test_pickleable.py b/Lib/test/test_email/test_pickleable.py --- a/Lib/test/test_email/test_pickleable.py +++ b/Lib/test/test_email/test_pickleable.py @@ -2,55 +2,86 @@ import textwrap import copy import pickle +import email +import email.message from email import policy -from email import message_from_string from email.headerregistry import HeaderRegistry from test.test_email import TestEmailBase class TestPickleCopyHeader(TestEmailBase): - unstructured = HeaderRegistry()('subject', 'this is a test') + header_factory = HeaderRegistry() - def test_deepcopy_unstructured(self): - h = copy.deepcopy(self.unstructured) - self.assertEqual(str(h), str(self.unstructured)) + unstructured = header_factory('subject', 'this is a test') - def test_pickle_unstructured(self): - p = pickle.dumps(self.unstructured) + def _test_deepcopy(self, name, value): + header = self.header_factory(name, value) + h = copy.deepcopy(header) + self.assertEqual(str(h), str(header)) + + def _test_pickle(self, name, value): + header = self.header_factory(name, value) + p = pickle.dumps(header) h = pickle.loads(p) - self.assertEqual(str(h), str(self.unstructured)) + self.assertEqual(str(h), str(header)) - address = HeaderRegistry()('from', 'frodo(a)mordor.net') + headers = ( + ('subject', 'this is a test'), + ('from', 'frodo(a)mordor.net'), + ('to', 'a: k(a)b.com, y(a)z.com;, j(a)f.com'), + ('date', 'Tue, 29 May 2012 09:24:26 +1000'), + ) - def test_deepcopy_address(self): - h = copy.deepcopy(self.address) - self.assertEqual(str(h), str(self.address)) + for header in headers: + locals()['test_deepcopy_'+header[0]] = ( + lambda self, header=header: + self._test_deepcopy(*header)) - def test_pickle_address(self): - p = pickle.dumps(self.address) - h = pickle.loads(p) - self.assertEqual(str(h), str(self.address)) + for header in headers: + locals()['test_pickle_'+header[0]] = ( + lambda self, header=header: + self._test_pickle(*header)) class TestPickleCopyMessage(TestEmailBase): - testmsg = message_from_string(textwrap.dedent("""\ - From: frodo(a)mordor.net - To: bilbo(a)underhill.org - Subject: help + msgs = {} - I think I forgot the ring. - """), policy=policy.default) + # Note: there will be no custom header objects in the parsed message. + msgs['parsed'] = email.message_from_string(textwrap.dedent("""\ + Date: Tue, 29 May 2012 09:24:26 +1000 + From: frodo(a)mordor.net + To: bilbo(a)underhill.org + Subject: help - def test_deepcopy(self): - msg2 = copy.deepcopy(self.testmsg) - self.assertEqual(msg2.as_string(), self.testmsg.as_string()) + I think I forgot the ring. + """), policy=policy.default) - def test_pickle(self): - p = pickle.dumps(self.testmsg) + msgs['created'] = email.message.Message(policy=policy.default) + msgs['created']['Date'] = 'Tue, 29 May 2012 09:24:26 +1000' + msgs['created']['From'] = 'frodo(a)mordor.net' + msgs['created']['To'] = 'bilbo(a)underhill.org' + msgs['created']['Subject'] = 'help' + msgs['created'].set_payload('I think I forgot the ring.') + + def _test_deepcopy(self, msg): + msg2 = copy.deepcopy(msg) + self.assertEqual(msg2.as_string(), msg.as_string()) + + def _test_pickle(self, msg): + p = pickle.dumps(msg) msg2 = pickle.loads(p) - self.assertEqual(msg2.as_string(), self.testmsg.as_string()) + self.assertEqual(msg2.as_string(), msg.as_string()) + for name, msg in msgs.items(): + locals()['test_deepcopy_'+name] = ( + lambda self, msg=msg: + self._test_deepcopy(msg)) + + for name, msg in msgs.items(): + locals()['test_pickle_'+name] = ( + lambda self, msg=msg: + self._test_pickle(msg)) if __name__ == '__main__': -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Regularize test_email/test_headerregistry's references to policy.
by r.david.murray May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/7c70eb6537b8 changeset: 77215:7c70eb6537b8 user: R David Murray <rdmurray(a)bitdance.com> date: Mon May 28 20:22:37 2012 -0400 summary: Regularize test_email/test_headerregistry's references to policy. files: Lib/test/test_email/test_headerregistry.py | 12 +++++----- 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -650,14 +650,14 @@ def test_short_unstructured(self): h = self.make_header('subject', 'this is a test') - self.assertEqual(h.fold(policy=self.policy), + self.assertEqual(h.fold(policy=policy.default), 'subject: this is a test\n') def test_long_unstructured(self): h = self.make_header('Subject', 'This is a long header ' 'line that will need to be folded into two lines ' 'and will demonstrate basic folding') - self.assertEqual(h.fold(policy=self.policy), + self.assertEqual(h.fold(policy=policy.default), 'Subject: This is a long header line that will ' 'need to be folded into two lines\n' ' and will demonstrate basic folding\n') @@ -676,11 +676,11 @@ def test_fold_unstructured_single_word(self): h = self.make_header('Subject', 'test') - self.assertEqual(h.fold(policy=self.policy), 'Subject: test\n') + self.assertEqual(h.fold(policy=policy.default), 'Subject: test\n') def test_fold_unstructured_short(self): h = self.make_header('Subject', 'test test test') - self.assertEqual(h.fold(policy=self.policy), + self.assertEqual(h.fold(policy=policy.default), 'Subject: test test test\n') def test_fold_unstructured_with_overlong_word(self): @@ -721,7 +721,7 @@ h = self.make_header('To', '"Theodore H. Perfect" <yes(a)man.com>, ' '"My address is very long because my name is long" <foo(a)bar.com>, ' '"Only A. Friend" <no(a)yes.com>') - self.assertEqual(h.fold(policy=self.policy), textwrap.dedent("""\ + self.assertEqual(h.fold(policy=policy.default), textwrap.dedent("""\ To: "Theodore H. Perfect" <yes(a)man.com>, "My address is very long because my name is long" <foo(a)bar.com>, "Only A. Friend" <no(a)yes.com> @@ -729,7 +729,7 @@ def test_fold_date_header(self): h = self.make_header('Date', 'Sat, 2 Feb 2002 17:00:06 -0800') - self.assertEqual(h.fold(policy=self.policy), + self.assertEqual(h.fold(policy=policy.default), 'Date: Sat, 02 Feb 2002 17:00:06 -0800\n') -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Refactor test_email/test_defect_handling.
by r.david.murray May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/fe426268eea6 changeset: 77214:fe426268eea6 user: R David Murray <rdmurray(a)bitdance.com> date: Mon May 28 20:14:10 2012 -0400 summary: Refactor test_email/test_defect_handling. files: Lib/test/test_email/test_defect_handling.py | 355 ++++----- Lib/test/test_email/test_email.py | 2 +- 2 files changed, 170 insertions(+), 187 deletions(-) diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py --- a/Lib/test/test_email/test_defect_handling.py +++ b/Lib/test/test_email/test_defect_handling.py @@ -1,84 +1,89 @@ import textwrap import unittest -from email._policybase import Compat32 +import contextlib +from email import policy from email import errors from test.test_email import TestEmailBase -class TestMessageDefectDetectionBase: +class TestDefectsBase: - dup_boundary_msg = textwrap.dedent("""\ - Subject: XX - From: xx(a)xx.dk - To: XX - Mime-version: 1.0 - Content-type: multipart/mixed; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + policy = policy.default + raise_expected = False - --MS_Mac_OE_3071477847_720252_MIME_Part - Content-type: multipart/alternative; - boundary="MS_Mac_OE_3071477847_720252_MIME_Part" - - --MS_Mac_OE_3071477847_720252_MIME_Part - Content-type: text/plain; charset="ISO-8859-1" - Content-transfer-encoding: quoted-printable - - text - - --MS_Mac_OE_3071477847_720252_MIME_Part - Content-type: text/html; charset="ISO-8859-1" - Content-transfer-encoding: quoted-printable - - <HTML></HTML> - - --MS_Mac_OE_3071477847_720252_MIME_Part-- - - --MS_Mac_OE_3071477847_720252_MIME_Part - Content-type: image/gif; name="xx.gif"; - Content-disposition: attachment - Content-transfer-encoding: base64 - - Some removed base64 encoded chars. - - --MS_Mac_OE_3071477847_720252_MIME_Part-- - - """) + @contextlib.contextmanager + def _raise_point(self, defect): + yield def test_same_boundary_inner_outer(self): + source = textwrap.dedent("""\ + Subject: XX + From: xx(a)xx.dk + To: XX + Mime-version: 1.0 + Content-type: multipart/mixed; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: multipart/alternative; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: text/plain; charset="ISO-8859-1" + Content-transfer-encoding: quoted-printable + + text + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: text/html; charset="ISO-8859-1" + Content-transfer-encoding: quoted-printable + + <HTML></HTML> + + --MS_Mac_OE_3071477847_720252_MIME_Part-- + + --MS_Mac_OE_3071477847_720252_MIME_Part + Content-type: image/gif; name="xx.gif"; + Content-disposition: attachment + Content-transfer-encoding: base64 + + Some removed base64 encoded chars. + + --MS_Mac_OE_3071477847_720252_MIME_Part-- + + """) # XXX better would be to actually detect the duplicate. - msg = self._str_msg(self.dup_boundary_msg) + with self._raise_point(errors.StartBoundaryNotFoundDefect): + msg = self._str_msg(source) + if self.raise_expected: return inner = msg.get_payload(0) self.assertTrue(hasattr(inner, 'defects')) self.assertEqual(len(self.get_defects(inner)), 1) self.assertTrue(isinstance(self.get_defects(inner)[0], errors.StartBoundaryNotFoundDefect)) - def test_same_boundary_inner_outer_raises_on_defect(self): - with self.assertRaises(errors.StartBoundaryNotFoundDefect): - self._str_msg(self.dup_boundary_msg, - policy=self.policy.clone(raise_on_defect=True)) + def test_multipart_no_boundary(self): + source = textwrap.dedent("""\ + Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) + From: foobar + Subject: broken mail + MIME-Version: 1.0 + Content-Type: multipart/report; report-type=delivery-status; - no_boundary_msg = textwrap.dedent("""\ - Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) - From: foobar - Subject: broken mail - MIME-Version: 1.0 - Content-Type: multipart/report; report-type=delivery-status; + --JAB03225.986577786/zinfandel.lacita.com - --JAB03225.986577786/zinfandel.lacita.com + One part - One part + --JAB03225.986577786/zinfandel.lacita.com + Content-Type: message/delivery-status - --JAB03225.986577786/zinfandel.lacita.com - Content-Type: message/delivery-status + Header: Another part - Header: Another part - - --JAB03225.986577786/zinfandel.lacita.com-- - """) - - def test_multipart_no_boundary(self): - msg = self._str_msg(self.no_boundary_msg) + --JAB03225.986577786/zinfandel.lacita.com-- + """) + with self._raise_point(errors.NoBoundaryInMultipartDefect): + msg = self._str_msg(source) + if self.raise_expected: return self.assertTrue(isinstance(msg.get_payload(), str)) self.assertEqual(len(self.get_defects(msg)), 2) self.assertTrue(isinstance(self.get_defects(msg)[0], @@ -86,11 +91,6 @@ self.assertTrue(isinstance(self.get_defects(msg)[1], errors.MultipartInvariantViolationDefect)) - def test_multipart_no_boundary_raise_on_defect(self): - with self.assertRaises(errors.NoBoundaryInMultipartDefect): - self._str_msg(self.no_boundary_msg, - policy=self.policy.clone(raise_on_defect=True)) - multipart_msg = textwrap.dedent("""\ Date: Wed, 14 Nov 2007 12:56:23 GMT From: foo(a)bar.invalid @@ -115,43 +115,42 @@ """) def test_multipart_invalid_cte(self): - msg = self._str_msg( - self.multipart_msg.format("\nContent-Transfer-Encoding: base64")) + with self._raise_point( + errors.InvalidMultipartContentTransferEncodingDefect): + msg = self._str_msg( + self.multipart_msg.format( + "\nContent-Transfer-Encoding: base64")) + if self.raise_expected: return self.assertEqual(len(self.get_defects(msg)), 1) self.assertIsInstance(self.get_defects(msg)[0], errors.InvalidMultipartContentTransferEncodingDefect) - def test_multipart_invalid_cte_raise_on_defect(self): - with self.assertRaises( - errors.InvalidMultipartContentTransferEncodingDefect): - self._str_msg( - self.multipart_msg.format( - "\nContent-Transfer-Encoding: base64"), - policy=self.policy.clone(raise_on_defect=True)) - def test_multipart_no_cte_no_defect(self): + if self.raise_expected: return msg = self._str_msg(self.multipart_msg.format('')) self.assertEqual(len(self.get_defects(msg)), 0) def test_multipart_valid_cte_no_defect(self): + if self.raise_expected: return for cte in ('7bit', '8bit', 'BINary'): msg = self._str_msg( self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte)) self.assertEqual(len(self.get_defects(msg)), 0, "cte="+cte) - lying_multipart_msg = textwrap.dedent("""\ - From: "Allison Dunlap" <xxx(a)example.com> - To: yyy(a)example.com - Subject: 64423 - Date: Sun, 11 Jul 2004 16:09:27 -0300 - MIME-Version: 1.0 - Content-Type: multipart/alternative; + def test_lying_multipart(self): + source = textwrap.dedent("""\ + From: "Allison Dunlap" <xxx(a)example.com> + To: yyy(a)example.com + Subject: 64423 + Date: Sun, 11 Jul 2004 16:09:27 -0300 + MIME-Version: 1.0 + Content-Type: multipart/alternative; - Blah blah blah - """) - - def test_lying_multipart(self): - msg = self._str_msg(self.lying_multipart_msg) + Blah blah blah + """) + with self._raise_point(errors.NoBoundaryInMultipartDefect): + msg = self._str_msg(source) + if self.raise_expected: return self.assertTrue(hasattr(msg, 'defects')) self.assertEqual(len(self.get_defects(msg)), 2) self.assertTrue(isinstance(self.get_defects(msg)[0], @@ -159,34 +158,28 @@ self.assertTrue(isinstance(self.get_defects(msg)[1], errors.MultipartInvariantViolationDefect)) - def test_lying_multipart_raise_on_defect(self): - with self.assertRaises(errors.NoBoundaryInMultipartDefect): - self._str_msg(self.lying_multipart_msg, - policy=self.policy.clone(raise_on_defect=True)) + def test_missing_start_boundary(self): + source = textwrap.dedent("""\ + Content-Type: multipart/mixed; boundary="AAA" + From: Mail Delivery Subsystem <xxx(a)example.com> + To: yyy(a)example.com - missing_start_boundary_msg = textwrap.dedent("""\ - Content-Type: multipart/mixed; boundary="AAA" - From: Mail Delivery Subsystem <xxx(a)example.com> - To: yyy(a)example.com + --AAA - --AAA + Stuff - Stuff + --AAA + Content-Type: message/rfc822 - --AAA - Content-Type: message/rfc822 + From: webmaster(a)python.org + To: zzz(a)example.com + Content-Type: multipart/mixed; boundary="BBB" - From: webmaster(a)python.org - To: zzz(a)example.com - Content-Type: multipart/mixed; boundary="BBB" + --BBB-- - --BBB-- + --AAA-- - --AAA-- - - """) - - def test_missing_start_boundary(self): + """) # The message structure is: # # multipart/mixed @@ -195,19 +188,18 @@ # multipart/mixed [*] # # [*] This message is missing its start boundary - outer = self._str_msg(self.missing_start_boundary_msg) + with self._raise_point(errors.StartBoundaryNotFoundDefect): + outer = self._str_msg(source) + if self.raise_expected: return bad = outer.get_payload(1).get_payload(0) self.assertEqual(len(self.get_defects(bad)), 1) self.assertTrue(isinstance(self.get_defects(bad)[0], errors.StartBoundaryNotFoundDefect)) - def test_missing_start_boundary_raise_on_defect(self): - with self.assertRaises(errors.StartBoundaryNotFoundDefect): - self._str_msg(self.missing_start_boundary_msg, - policy=self.policy.clone(raise_on_defect=True)) - def test_first_line_is_continuation_header(self): - msg = self._str_msg(' Line 1\nSubject: test\n\nbody') + with self._raise_point(errors.FirstHeaderLineIsContinuationDefect): + msg = self._str_msg(' Line 1\nSubject: test\n\nbody') + if self.raise_expected: return self.assertEqual(msg.keys(), ['Subject']) self.assertEqual(msg.get_payload(), 'body') self.assertEqual(len(self.get_defects(msg)), 1) @@ -215,113 +207,92 @@ [errors.FirstHeaderLineIsContinuationDefect]) self.assertEqual(self.get_defects(msg)[0].line, ' Line 1\n') - def test_first_line_is_continuation_header_raise_on_defect(self): - with self.assertRaises(errors.FirstHeaderLineIsContinuationDefect): - self._str_msg(' Line 1\nSubject: test\n\nbody\n', - policy=self.policy.clone(raise_on_defect=True)) - def test_missing_header_body_separator(self): # Our heuristic if we see a line that doesn't look like a header (no # leading whitespace but no ':') is to assume that the blank line that # separates the header from the body is missing, and to stop parsing # headers and start parsing the body. - msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') + with self._raise_point(errors.MissingHeaderBodySeparatorDefect): + msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') + if self.raise_expected: return self.assertEqual(msg.keys(), ['Subject']) self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') self.assertDefectsEqual(self.get_defects(msg), [errors.MissingHeaderBodySeparatorDefect]) - def test_missing_header_body_separator_raise_on_defect(self): - with self.assertRaises(errors.MissingHeaderBodySeparatorDefect): - self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n', - policy=self.policy.clone(raise_on_defect=True)) + def test_bad_padding_in_base64_payload(self): + source = textwrap.dedent("""\ + Subject: test + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 - badly_padded_base64_payload = textwrap.dedent("""\ - Subject: test - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - - dmk - """) - - def test_bad_padding_in_base64_payload(self): - msg = self._str_msg(self.badly_padded_base64_payload) - self.assertEqual(msg.get_payload(decode=True), b'vi') + dmk + """) + msg = self._str_msg(source) + with self._raise_point(errors.InvalidBase64PaddingDefect): + payload = msg.get_payload(decode=True) + if self.raise_expected: return + self.assertEqual(payload, b'vi') self.assertDefectsEqual(self.get_defects(msg), [errors.InvalidBase64PaddingDefect]) - def test_bad_padding_in_base64_payload_raise_on_defect(self): - msg = self._str_msg(self.badly_padded_base64_payload, - policy=self.policy.clone(raise_on_defect=True)) - with self.assertRaises(errors.InvalidBase64PaddingDefect): - msg.get_payload(decode=True) + def test_invalid_chars_in_base64_payload(self): + source = textwrap.dedent("""\ + Subject: test + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 - invalid_chars_in_base64_payload = textwrap.dedent("""\ - Subject: test - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - - dm\x01k=== - """) - - def test_invalid_chars_in_base64_payload(self): - msg = self._str_msg(self.invalid_chars_in_base64_payload) - self.assertEqual(msg.get_payload(decode=True), b'vi') + dm\x01k=== + """) + msg = self._str_msg(source) + with self._raise_point(errors.InvalidBase64CharactersDefect): + payload = msg.get_payload(decode=True) + if self.raise_expected: return + self.assertEqual(payload, b'vi') self.assertDefectsEqual(self.get_defects(msg), [errors.InvalidBase64CharactersDefect]) - def test_invalid_chars_in_base64_payload_raise_on_defect(self): - msg = self._str_msg(self.invalid_chars_in_base64_payload, - policy=self.policy.clone(raise_on_defect=True)) - with self.assertRaises(errors.InvalidBase64CharactersDefect): - msg.get_payload(decode=True) + def test_missing_ending_boundary(self): + source = textwrap.dedent("""\ + To: 1(a)harrydomain4.com + Subject: Fwd: 1 + MIME-Version: 1.0 + Content-Type: multipart/alternative; + boundary="------------000101020201080900040301" - missing_ending_boundary = textwrap.dedent("""\ - To: 1(a)harrydomain4.com - Subject: Fwd: 1 - MIME-Version: 1.0 - Content-Type: multipart/alternative; - boundary="------------000101020201080900040301" + --------------000101020201080900040301 + Content-Type: text/plain; charset=ISO-8859-1 + Content-Transfer-Encoding: 7bit - --------------000101020201080900040301 - Content-Type: text/plain; charset=ISO-8859-1 - Content-Transfer-Encoding: 7bit + Alternative 1 - Alternative 1 + --------------000101020201080900040301 + Content-Type: text/html; charset=ISO-8859-1 + Content-Transfer-Encoding: 7bit - --------------000101020201080900040301 - Content-Type: text/html; charset=ISO-8859-1 - Content-Transfer-Encoding: 7bit + Alternative 2 - Alternative 2 - - """) - - def test_missing_ending_boundary(self): - msg = self._str_msg(self.missing_ending_boundary) + """) + with self._raise_point(errors.CloseBoundaryNotFoundDefect): + msg = self._str_msg(source) + if self.raise_expected: return self.assertEqual(len(msg.get_payload()), 2) self.assertEqual(msg.get_payload(1).get_payload(), 'Alternative 2\n') self.assertDefectsEqual(self.get_defects(msg), [errors.CloseBoundaryNotFoundDefect]) - def test_missing_ending_boundary_raise_on_defect(self): - with self.assertRaises(errors.CloseBoundaryNotFoundDefect): - self._str_msg(self.missing_ending_boundary, - policy=self.policy.clone(raise_on_defect=True)) - -class TestMessageDefectDetection(TestMessageDefectDetectionBase, TestEmailBase): +class TestDefectDetection(TestDefectsBase, TestEmailBase): def get_defects(self, obj): return obj.defects -class TestMessageDefectDetectionCapture(TestMessageDefectDetectionBase, - TestEmailBase): +class TestDefectCapture(TestDefectsBase, TestEmailBase): - class CapturePolicy(Compat32): + class CapturePolicy(policy.EmailPolicy): captured = None def register_defect(self, obj, defect): self.captured.append(defect) @@ -333,5 +304,17 @@ return self.policy.captured +class TestDefectRaising(TestDefectsBase, TestEmailBase): + + policy = TestDefectsBase.policy + policy = policy.clone(raise_on_defect=True) + raise_expected = True + + @contextlib.contextmanager + def _raise_point(self, defect): + with self.assertRaises(defect): + yield + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -1973,7 +1973,7 @@ [errors.FirstHeaderLineIsContinuationDefect]) eq(msg.defects[0].line, ' Line 1\n') - # test_parser.TestMessageDefectDetectionBase + # test_defect_handling def test_missing_header_body_separator(self): # Our heuristic if we see a line that doesn't look like a header (no # leading whitespace but no ':') is to assume that the blank line that -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Issue #14930: Make memoryview objects weakrefable.
by richard.oudkerk May 28, 2012

May 28, 2012

http://hg.python.org/cpython/rev/bc0281f85409 changeset: 77213:bc0281f85409 parent: 77211:7951900afd00 user: Richard Oudkerk <shibturn(a)gmail.com> date: Mon May 28 21:35:09 2012 +0100 summary: Issue #14930: Make memoryview objects weakrefable. files: Include/memoryobject.h | 1 + Lib/test/test_memoryview.py | 15 +++++++++++++++ Lib/test/test_sys.py | 2 +- Misc/NEWS | 2 ++ Objects/memoryobject.c | 5 ++++- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Include/memoryobject.h b/Include/memoryobject.h --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -63,6 +63,7 @@ Py_ssize_t exports; /* number of buffer re-exports */ Py_buffer view; /* private copy of the exporter's view */ char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* used for casting */ + PyObject *weakreflist; Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */ } PyMemoryViewObject; #endif diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -336,6 +336,21 @@ m = self._view(b) self.assertRaises(ValueError, hash, m) + def test_weakref(self): + # Check memoryviews are weakrefable + for tp in self._types: + b = tp(self._source) + m = self._view(b) + L = [] + def callback(wr, b=b): + L.append(b) + wr = weakref.ref(m, callback) + self.assertIs(wr(), m) + del m + test.support.gc_collect() + self.assertIs(wr(), None) + self.assertIs(L[0], b) + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. # NOTE: support for multi-dimensional objects is unimplemented. diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -778,7 +778,7 @@ check(int(PyLong_BASE**2-1), size(vh) + 2*self.longdigit) check(int(PyLong_BASE**2), size(vh) + 3*self.longdigit) # memoryview - check(memoryview(b''), size(h + 'PPiP4P2i5P3cP')) + check(memoryview(b''), size(h + 'PPiP4P2i5P3c2P')) # module check(unittest, size(h + '3P')) # None diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #14930: Make memoryview objects weakrefable. + - Issue #14775: Fix a potential quadratic dict build-up due to the garbage collector repeatedly trying to untrack dicts. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -595,6 +595,7 @@ mv->view.shape = mv->ob_array; mv->view.strides = mv->ob_array + ndim; mv->view.suboffsets = mv->ob_array + 2 * ndim; + mv->weakreflist = NULL; _PyObject_GC_TRACK(mv); return mv; @@ -969,6 +970,8 @@ _PyObject_GC_UNTRACK(self); (void)_memory_release(self); Py_CLEAR(self->mbuf); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); PyObject_GC_Del(self); } @@ -2608,7 +2611,7 @@ (traverseproc)memory_traverse, /* tp_traverse */ (inquiry)memory_clear, /* tp_clear */ memory_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ + offsetof(PyMemoryViewObject, weakreflist),/* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ memory_methods, /* tp_methods */ -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Issue #14775: Fix a potential quadratic dict build-up due to the garbage
by antoine.pitrou May 28, 2012

May 28, 2012

http://hg.python.org/cpython/rev/6554ebbeb2f3 changeset: 77212:6554ebbeb2f3 branch: 2.7 parent: 77207:8a65eceea56c user: Antoine Pitrou <solipsis(a)pitrou.net> date: Mon May 28 22:22:34 2012 +0200 summary: Issue #14775: Fix a potential quadratic dict build-up due to the garbage collector repeatedly trying to untrack dicts. Additional comments by Tim Silk. files: Misc/ACKS | 1 + Misc/NEWS | 3 + Modules/gcmodule.c | 60 ++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -791,6 +791,7 @@ Itamar Shtull-Trauring Eric Siegerman Paul Sijben +Tim Silk Kirill Simonov Nathan Paul Simons Janne Sinkkonen diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ Core and Builtins ----------------- +- Issue #14775: Fix a potential quadratic dict build-up due to the garbage + collector repeatedly trying to untrack dicts. + - Issue #14494: Fix __future__.py and its documentation to note that absolute imports are the default behavior in 3.0 instead of 2.7. Patch by Sven Marnach. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -111,6 +111,46 @@ http://mail.python.org/pipermail/python-dev/2008-June/080579.html */ +/* + NOTE: about untracking of mutable objects. + + Certain types of container cannot participate in a reference cycle, and + so do not need to be tracked by the garbage collector. Untracking these + objects reduces the cost of garbage collections. However, determining + which objects may be untracked is not free, and the costs must be + weighed against the benefits for garbage collection. + + There are two possible strategies for when to untrack a container: + + i) When the container is created. + ii) When the container is examined by the garbage collector. + + Tuples containing only immutable objects (integers, strings etc, and + recursively, tuples of immutable objects) do not need to be tracked. + The interpreter creates a large number of tuples, many of which will + not survive until garbage collection. It is therefore not worthwhile + to untrack eligible tuples at creation time. + + Instead, all tuples except the empty tuple are tracked when created. + During garbage collection it is determined whether any surviving tuples + can be untracked. A tuple can be untracked if all of its contents are + already not tracked. Tuples are examined for untracking in all garbage + collection cycles. It may take more than one cycle to untrack a tuple. + + Dictionaries containing only immutable objects also do not need to be + tracked. Dictionaries are untracked when created. If a tracked item is + inserted into a dictionary (either as a key or value), the dictionary + becomes tracked. During a full garbage collection (all generations), + the collector will untrack any dictionaries whose contents are not + tracked. + + The module provides the python function is_tracked(obj), which returns + the CURRENT tracking status of the object. Subsequent garbage + collections may change the tracking status of the object. + + Untracking of certain containers was introduced in issue #4688, and + the algorithm was refined in response to issue #14775. +*/ /* set for debugging information */ #define DEBUG_STATS (1<<0) /* print collection statistics */ @@ -436,9 +476,6 @@ if (PyTuple_CheckExact(op)) { _PyTuple_MaybeUntrack(op); } - else if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - } } else { /* This *may* be unreachable. To make progress, @@ -478,6 +515,20 @@ return 0; } +/* Try to untrack all currently tracked dictionaries */ +static void +untrack_dicts(PyGC_Head *head) +{ + PyGC_Head *next, *gc = head->gc.gc_next; + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = gc->gc.gc_next; + if (PyDict_CheckExact(op)) + _PyDict_MaybeUntrack(op); + gc = next; + } +} + /* Move the objects in unreachable with __del__ methods into `finalizers`. * Objects moved into `finalizers` have gc_refs set to GC_REACHABLE; the * objects remaining in unreachable are left at GC_TENTATIVELY_UNREACHABLE. @@ -890,6 +941,9 @@ gc_list_merge(young, old); } else { + /* We only untrack dicts in full collections, to avoid quadratic + dict build-up. See issue #14775. */ + untrack_dicts(young); long_lived_pending = 0; long_lived_total = gc_list_size(young); } -- Repository URL: http://hg.python.org/cpython

1 0