Mailman 3 June 2015 - Python-checkins

Daily reference leaks (d792dc240456): sum=4
by solipsis＠pitrou.net June 30, 2015

June 30, 2015

results for d792dc240456 on branch "default" -------------------------------------------- test_functools leaked [0, 2, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogoqUTGv', '--timeout', '7200']

1 0

cpython (merge 3.5 -> default): Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree.
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/d792dc240456 changeset: 96723:d792dc240456 parent: 96720:97a24bc714ec parent: 96722:6ae8842e9b60 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 23:11:51 2015 +0300 summary: Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. A deprecation warning no longer issued by XMLParser subclass with default doctype() method. Direct call of doctype() now issues a warning. Parser's doctype() … [View More]now is not called if target's doctype() is called. Based on patch by Martin Panter. files: Lib/test/test_xml_etree.py | 41 +++++++++++++++++ Misc/NEWS | 6 ++ Modules/_elementtree.c | 50 ++++++++++++++------ Modules/clinic/_elementtree.c.h | 25 +++++++-- 4 files changed, 100 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -12,6 +12,7 @@ import sys import types import unittest +import warnings import weakref from itertools import product @@ -2237,6 +2238,20 @@ parser.feed(self.sample1) self._check_sample_element(parser.close()) + def test_doctype_warning(self): + parser = ET.XMLParser() + with self.assertWarns(DeprecationWarning): + parser.doctype('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd') + parser.feed('<html/>') + parser.close() + + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + parser = ET.XMLParser() + parser.feed(self.sample2) + parser.close() + def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -2252,6 +2267,32 @@ ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + _doctype = _doctype2 = None + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class DoctypeParser: + def doctype(self, name, pubid, system): + nonlocal _doctype2 + _doctype2 = (name, pubid, system) + + parser = MyParserWithDoctype(target=DoctypeParser()) + parser.feed(self.sample2) + parser.close() + self.assertIsNone(_doctype) + self.assertEqual(_doctype2, + ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + + def test_inherited_doctype(self): + '''Ensure that ordinary usage is not deprecated (Issue 19176)''' + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class MyParserWithoutDoctype(ET.XMLParser): + pass + parser = MyParserWithoutDoctype() + parser.feed(self.sample2) + parser.close() + def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,12 @@ Library ------- +- Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. + A deprecation warning no longer issued by XMLParser subclass with default + doctype() method. Direct call of doctype() now issues a warning. Parser's + doctype() now is not called if target's doctype() is called. Based on patch + by Martin Panter. + - Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize for tab-indented blocks. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2782,7 +2782,11 @@ } XMLParserObject; -#define XMLParser_CheckExact(op) (Py_TYPE(op) == &XMLParser_Type) +static PyObject* +_elementtree_XMLParser_doctype(XMLParserObject* self, PyObject* args); +static PyObject * +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system); /* helpers */ @@ -3182,20 +3186,22 @@ doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); } - - /* Now see if the parser itself has a doctype method. If yes and it's - * a subclass, call it but warn about deprecation. If it's not a subclass - * (i.e. vanilla XMLParser), do nothing. - */ - parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); - if (parser_doctype) { - if (!XMLParser_CheckExact(self_pyobj)) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "This method of XMLParser is deprecated. Define" - " doctype() method on the TreeBuilder target.", - 1) < 0) { + else { + /* Now see if the parser itself has a doctype method. If yes and it's + * a custom method, call it but warn about deprecation. If it's only + * the vanilla XMLParser method, do nothing. + */ + parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); + if (parser_doctype && + !(PyCFunction_Check(parser_doctype) && + PyCFunction_GET_SELF(parser_doctype) == self_pyobj && + PyCFunction_GET_FUNCTION(parser_doctype) == + (PyCFunction) _elementtree_XMLParser_doctype)) { + res = _elementtree_XMLParser_doctype_impl(self, doctype_name_obj, + pubid_obj, sysid_obj); + if (!res) goto clear; - } + Py_DECREF(res); res = PyObject_CallFunction(parser_doctype, "OOO", doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); @@ -3572,12 +3578,24 @@ /*[clinic input] _elementtree.XMLParser.doctype + name: object + pubid: object + system: object + / + [clinic start generated code]*/ static PyObject * -_elementtree_XMLParser_doctype_impl(XMLParserObject *self) -/*[clinic end generated code: output=d09fdb9c45f3a602 input=20d5e0febf902a2f]*/ +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system) +/*[clinic end generated code: output=10fb50c2afded88d input=84050276cca045e1]*/ { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "This method of XMLParser is deprecated. Define" + " doctype() method on the TreeBuilder target.", + 1) < 0) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -619,20 +619,33 @@ {"_parse_whole", (PyCFunction)_elementtree_XMLParser__parse_whole, METH_O, _elementtree_XMLParser__parse_whole__doc__}, PyDoc_STRVAR(_elementtree_XMLParser_doctype__doc__, -"doctype($self, /)\n" +"doctype($self, name, pubid, system, /)\n" "--\n" "\n"); #define _ELEMENTTREE_XMLPARSER_DOCTYPE_METHODDEF \ - {"doctype", (PyCFunction)_elementtree_XMLParser_doctype, METH_NOARGS, _elementtree_XMLParser_doctype__doc__}, + {"doctype", (PyCFunction)_elementtree_XMLParser_doctype, METH_VARARGS, _elementtree_XMLParser_doctype__doc__}, static PyObject * -_elementtree_XMLParser_doctype_impl(XMLParserObject *self); +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system); static PyObject * -_elementtree_XMLParser_doctype(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_XMLParser_doctype(XMLParserObject *self, PyObject *args) { - return _elementtree_XMLParser_doctype_impl(self); + PyObject *return_value = NULL; + PyObject *name; + PyObject *pubid; + PyObject *system; + + if (!PyArg_UnpackTuple(args, "doctype", + 3, 3, + &name, &pubid, &system)) + goto exit; + return_value = _elementtree_XMLParser_doctype_impl(self, name, pubid, system); + +exit: + return return_value; } PyDoc_STRVAR(_elementtree_XMLParser__setevents__doc__, @@ -663,4 +676,4 @@ exit: return return_value; } -/*[clinic end generated code: output=119aed84c1545187 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=25b8bf7e7f2151ca input=a9049054013a1b77]*/ -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (3.4): Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree.
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/75571407dcd3 changeset: 96721:75571407dcd3 branch: 3.4 parent: 96718:c95d7ffa492e user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 22:35:58 2015 +0300 summary: Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. A deprecation warning no longer issued by XMLParser subclass with default doctype() method. Direct call of doctype() now issues a warning. Parser's doctype() now is not called … [View More]if target's doctype() is called. Based on patch by Martin Panter. files: Lib/test/test_xml_etree.py | 41 ++++++++++++++++++++++++++ Misc/NEWS | 6 +++ Modules/_elementtree.c | 35 +++++++++++++-------- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -12,6 +12,7 @@ import sys import types import unittest +import warnings import weakref from itertools import product @@ -2237,6 +2238,20 @@ parser.feed(self.sample1) self._check_sample_element(parser.close()) + def test_doctype_warning(self): + parser = ET.XMLParser() + with self.assertWarns(DeprecationWarning): + parser.doctype('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd') + parser.feed('<html/>') + parser.close() + + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + parser = ET.XMLParser() + parser.feed(self.sample2) + parser.close() + def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -2252,6 +2267,32 @@ ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + _doctype = _doctype2 = None + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class DoctypeParser: + def doctype(self, name, pubid, system): + nonlocal _doctype2 + _doctype2 = (name, pubid, system) + + parser = MyParserWithDoctype(target=DoctypeParser()) + parser.feed(self.sample2) + parser.close() + self.assertIsNone(_doctype) + self.assertEqual(_doctype2, + ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + + def test_inherited_doctype(self): + '''Ensure that ordinary usage is not deprecated (Issue 19176)''' + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class MyParserWithoutDoctype(ET.XMLParser): + pass + parser = MyParserWithoutDoctype() + parser.feed(self.sample2) + parser.close() + def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -64,6 +64,12 @@ Library ------- +- Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. + A deprecation warning no longer issued by XMLParser subclass with default + doctype() method. Direct call of doctype() now issues a warning. Parser's + doctype() now is not called if target's doctype() is called. Based on patch + by Martin Panter. + - Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize for tab-indented blocks. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2791,7 +2791,7 @@ } XMLParserObject; -#define XMLParser_CheckExact(op) (Py_TYPE(op) == &XMLParser_Type) +static PyObject* xmlparser_doctype(XMLParserObject* self, PyObject* args); /* helpers */ @@ -3190,20 +3190,21 @@ doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); } - - /* Now see if the parser itself has a doctype method. If yes and it's - * a subclass, call it but warn about deprecation. If it's not a subclass - * (i.e. vanilla XMLParser), do nothing. - */ - parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); - if (parser_doctype) { - if (!XMLParser_CheckExact(self_pyobj)) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "This method of XMLParser is deprecated. Define" - " doctype() method on the TreeBuilder target.", - 1) < 0) { + else { + /* Now see if the parser itself has a doctype method. If yes and it's + * a custom method, call it but warn about deprecation. If it's only + * the vanilla XMLParser method, do nothing. + */ + parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); + if (parser_doctype && + !(PyCFunction_Check(parser_doctype) && + PyCFunction_GET_SELF(parser_doctype) == self_pyobj && + PyCFunction_GET_FUNCTION(parser_doctype) == + (PyCFunction) xmlparser_doctype)) { + res = xmlparser_doctype(self, NULL); + if (!res) goto clear; - } + Py_DECREF(res); res = PyObject_CallFunction(parser_doctype, "OOO", doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); @@ -3556,6 +3557,12 @@ static PyObject* xmlparser_doctype(XMLParserObject *self, PyObject *args) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "This method of XMLParser is deprecated. Define" + " doctype() method on the TreeBuilder target.", + 1) < 0) { + return NULL; + } Py_RETURN_NONE; } -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge 3.4 -> 3.5): Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree.
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/6ae8842e9b60 changeset: 96722:6ae8842e9b60 branch: 3.5 parent: 96719:942860bada14 parent: 96721:75571407dcd3 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 23:08:52 2015 +0300 summary: Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. A deprecation warning no longer issued by XMLParser subclass with default doctype() method. Direct call of doctype() now issues a warning. … [View More]Parser's doctype() now is not called if target's doctype() is called. Based on patch by Martin Panter. files: Lib/test/test_xml_etree.py | 41 +++++++++++++++++ Misc/NEWS | 6 ++ Modules/_elementtree.c | 50 ++++++++++++++------ Modules/clinic/_elementtree.c.h | 25 +++++++-- 4 files changed, 100 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -12,6 +12,7 @@ import sys import types import unittest +import warnings import weakref from itertools import product @@ -2237,6 +2238,20 @@ parser.feed(self.sample1) self._check_sample_element(parser.close()) + def test_doctype_warning(self): + parser = ET.XMLParser() + with self.assertWarns(DeprecationWarning): + parser.doctype('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd') + parser.feed('<html/>') + parser.close() + + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + parser = ET.XMLParser() + parser.feed(self.sample2) + parser.close() + def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -2252,6 +2267,32 @@ ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + _doctype = _doctype2 = None + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class DoctypeParser: + def doctype(self, name, pubid, system): + nonlocal _doctype2 + _doctype2 = (name, pubid, system) + + parser = MyParserWithDoctype(target=DoctypeParser()) + parser.feed(self.sample2) + parser.close() + self.assertIsNone(_doctype) + self.assertEqual(_doctype2, + ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + + def test_inherited_doctype(self): + '''Ensure that ordinary usage is not deprecated (Issue 19176)''' + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + class MyParserWithoutDoctype(ET.XMLParser): + pass + parser = MyParserWithoutDoctype() + parser.feed(self.sample2) + parser.close() + def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,12 @@ Library ------- +- Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. + A deprecation warning no longer issued by XMLParser subclass with default + doctype() method. Direct call of doctype() now issues a warning. Parser's + doctype() now is not called if target's doctype() is called. Based on patch + by Martin Panter. + - Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize for tab-indented blocks. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2782,7 +2782,11 @@ } XMLParserObject; -#define XMLParser_CheckExact(op) (Py_TYPE(op) == &XMLParser_Type) +static PyObject* +_elementtree_XMLParser_doctype(XMLParserObject* self, PyObject* args); +static PyObject * +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system); /* helpers */ @@ -3182,20 +3186,22 @@ doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); } - - /* Now see if the parser itself has a doctype method. If yes and it's - * a subclass, call it but warn about deprecation. If it's not a subclass - * (i.e. vanilla XMLParser), do nothing. - */ - parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); - if (parser_doctype) { - if (!XMLParser_CheckExact(self_pyobj)) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "This method of XMLParser is deprecated. Define" - " doctype() method on the TreeBuilder target.", - 1) < 0) { + else { + /* Now see if the parser itself has a doctype method. If yes and it's + * a custom method, call it but warn about deprecation. If it's only + * the vanilla XMLParser method, do nothing. + */ + parser_doctype = PyObject_GetAttrString(self_pyobj, "doctype"); + if (parser_doctype && + !(PyCFunction_Check(parser_doctype) && + PyCFunction_GET_SELF(parser_doctype) == self_pyobj && + PyCFunction_GET_FUNCTION(parser_doctype) == + (PyCFunction) _elementtree_XMLParser_doctype)) { + res = _elementtree_XMLParser_doctype_impl(self, doctype_name_obj, + pubid_obj, sysid_obj); + if (!res) goto clear; - } + Py_DECREF(res); res = PyObject_CallFunction(parser_doctype, "OOO", doctype_name_obj, pubid_obj, sysid_obj); Py_CLEAR(res); @@ -3572,12 +3578,24 @@ /*[clinic input] _elementtree.XMLParser.doctype + name: object + pubid: object + system: object + / + [clinic start generated code]*/ static PyObject * -_elementtree_XMLParser_doctype_impl(XMLParserObject *self) -/*[clinic end generated code: output=d09fdb9c45f3a602 input=20d5e0febf902a2f]*/ +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system) +/*[clinic end generated code: output=10fb50c2afded88d input=84050276cca045e1]*/ { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "This method of XMLParser is deprecated. Define" + " doctype() method on the TreeBuilder target.", + 1) < 0) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -619,20 +619,33 @@ {"_parse_whole", (PyCFunction)_elementtree_XMLParser__parse_whole, METH_O, _elementtree_XMLParser__parse_whole__doc__}, PyDoc_STRVAR(_elementtree_XMLParser_doctype__doc__, -"doctype($self, /)\n" +"doctype($self, name, pubid, system, /)\n" "--\n" "\n"); #define _ELEMENTTREE_XMLPARSER_DOCTYPE_METHODDEF \ - {"doctype", (PyCFunction)_elementtree_XMLParser_doctype, METH_NOARGS, _elementtree_XMLParser_doctype__doc__}, + {"doctype", (PyCFunction)_elementtree_XMLParser_doctype, METH_VARARGS, _elementtree_XMLParser_doctype__doc__}, static PyObject * -_elementtree_XMLParser_doctype_impl(XMLParserObject *self); +_elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, + PyObject *pubid, PyObject *system); static PyObject * -_elementtree_XMLParser_doctype(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_XMLParser_doctype(XMLParserObject *self, PyObject *args) { - return _elementtree_XMLParser_doctype_impl(self); + PyObject *return_value = NULL; + PyObject *name; + PyObject *pubid; + PyObject *system; + + if (!PyArg_UnpackTuple(args, "doctype", + 3, 3, + &name, &pubid, &system)) + goto exit; + return_value = _elementtree_XMLParser_doctype_impl(self, name, pubid, system); + +exit: + return return_value; } PyDoc_STRVAR(_elementtree_XMLParser__setevents__doc__, @@ -663,4 +676,4 @@ exit: return return_value; } -/*[clinic end generated code: output=119aed84c1545187 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=25b8bf7e7f2151ca input=a9049054013a1b77]*/ -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge 3.5 -> default): Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/97a24bc714ec changeset: 96720:97a24bc714ec parent: 96714:f2f5d1c928eb parent: 96719:942860bada14 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 21:18:55 2015 +0300 summary: Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray object now always allocates place for trailing null byte and it's buffer now is always null-terminated. files: Lib/test/test_bytes.py | 19 ++++++++++++++++++- … [View More]Misc/NEWS | 4 ++++ Objects/bytearrayobject.c | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1098,10 +1098,27 @@ for i in range(100): b += b"x" alloc = b.__alloc__() - self.assertTrue(alloc >= len(b)) + self.assertGreater(alloc, len(b)) # including trailing null byte if alloc not in seq: seq.append(alloc) + def test_init_alloc(self): + b = bytearray() + def g(): + for i in range(1, 100): + yield i + a = list(b) + self.assertEqual(a, list(range(1, len(a)+1))) + self.assertEqual(len(b), len(a)) + self.assertLessEqual(len(b), i) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) # including trailing null byte + b.__init__(g()) + self.assertEqual(list(b), list(range(1, 100))) + self.assertEqual(len(b), 99) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) + def test_extend(self): orig = b'hello' a = bytearray(orig) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,10 @@ Core and Builtins ----------------- +- Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray + object now always allocates place for trailing null byte and it's buffer now + is always null-terminated. + - Upgrade to Unicode 8.0.0. - Issue #24345: Add Py_tp_finalize slot for the stable ABI. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -891,8 +891,10 @@ goto error; /* Append the byte */ - if (Py_SIZE(self) < self->ob_alloc) + if (Py_SIZE(self) + 1 < self->ob_alloc) { Py_SIZE(self)++; + PyByteArray_AS_STRING(self)[Py_SIZE(self)] = '\0'; + } else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) goto error; PyByteArray_AS_STRING(self)[Py_SIZE(self)-1] = value; -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (merge 3.4 -> 3.5): Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/942860bada14 changeset: 96719:942860bada14 branch: 3.5 parent: 96713:98380a6e037c parent: 96718:c95d7ffa492e user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 21:18:01 2015 +0300 summary: Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray object now always allocates place for trailing null byte and it's buffer now is always null-terminated. files: Lib/test/test_bytes.py | 19 +++++… [View More]+++++++++++++- Misc/NEWS | 4 ++++ Objects/bytearrayobject.c | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1098,10 +1098,27 @@ for i in range(100): b += b"x" alloc = b.__alloc__() - self.assertTrue(alloc >= len(b)) + self.assertGreater(alloc, len(b)) # including trailing null byte if alloc not in seq: seq.append(alloc) + def test_init_alloc(self): + b = bytearray() + def g(): + for i in range(1, 100): + yield i + a = list(b) + self.assertEqual(a, list(range(1, len(a)+1))) + self.assertEqual(len(b), len(a)) + self.assertLessEqual(len(b), i) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) # including trailing null byte + b.__init__(g()) + self.assertEqual(list(b), list(range(1, 100))) + self.assertEqual(len(b), 99) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) + def test_extend(self): orig = b'hello' a = bytearray(orig) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray + object now always allocates place for trailing null byte and it's buffer now + is always null-terminated. + - Upgrade to Unicode 8.0.0. - Issue #24345: Add Py_tp_finalize slot for the stable ABI. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -891,8 +891,10 @@ goto error; /* Append the byte */ - if (Py_SIZE(self) < self->ob_alloc) + if (Py_SIZE(self) + 1 < self->ob_alloc) { Py_SIZE(self)++; + PyByteArray_AS_STRING(self)[Py_SIZE(self)] = '\0'; + } else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) goto error; PyByteArray_AS_STRING(self)[Py_SIZE(self)-1] = value; -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (2.7): Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/a887ce8611d2 changeset: 96717:a887ce8611d2 branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 21:13:54 2015 +0300 summary: Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray object now always allocates place for trailing null byte and it's buffer now is always null-terminated. files: Lib/test/test_bytes.py | 19 ++++++++++++++++++- Misc/NEWS | 4 ++++ Objects/… [View More]bytearrayobject.c | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -722,10 +722,27 @@ for i in range(100): b += b"x" alloc = b.__alloc__() - self.assertTrue(alloc >= len(b)) + self.assertGreater(alloc, len(b)) # including trailing null byte if alloc not in seq: seq.append(alloc) + def test_init_alloc(self): + b = bytearray() + def g(): + for i in range(1, 100): + yield i + a = list(b) + self.assertEqual(a, list(range(1, len(a)+1))) + self.assertEqual(len(b), len(a)) + self.assertLessEqual(len(b), i) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) # including trailing null byte + b.__init__(g()) + self.assertEqual(list(b), list(range(1, 100))) + self.assertEqual(len(b), 99) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) + def test_extend(self): orig = b'hello' a = bytearray(orig) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray + object now always allocates place for trailing null byte and it's buffer now + is always null-terminated. + - Issue #19543: encode() and decode() methods and constructors of str, unicode and bytearray classes now emit deprecation warning for known non-text encodings when Python is ran with the -3 option. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -897,8 +897,10 @@ goto error; /* Append the byte */ - if (Py_SIZE(self) < self->ob_alloc) + if (Py_SIZE(self) + 1 < self->ob_alloc) { Py_SIZE(self)++; + PyByteArray_AS_STRING(self)[Py_SIZE(self)] = '\0'; + } else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) goto error; self->ob_bytes[Py_SIZE(self)-1] = value; -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython (3.4): Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray
by serhiy.storchaka June 29, 2015

June 29, 2015

https://hg.python.org/cpython/rev/c95d7ffa492e changeset: 96718:c95d7ffa492e branch: 3.4 parent: 96712:9ce5c1f371f7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Jun 29 21:14:06 2015 +0300 summary: Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray object now always allocates place for trailing null byte and it's buffer now is always null-terminated. files: Lib/test/test_bytes.py | 19 ++++++++++++++++++- Misc/NEWS … [View More] | 4 ++++ Objects/bytearrayobject.c | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1030,10 +1030,27 @@ for i in range(100): b += b"x" alloc = b.__alloc__() - self.assertTrue(alloc >= len(b)) + self.assertGreater(alloc, len(b)) # including trailing null byte if alloc not in seq: seq.append(alloc) + def test_init_alloc(self): + b = bytearray() + def g(): + for i in range(1, 100): + yield i + a = list(b) + self.assertEqual(a, list(range(1, len(a)+1))) + self.assertEqual(len(b), len(a)) + self.assertLessEqual(len(b), i) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) # including trailing null byte + b.__init__(g()) + self.assertEqual(list(b), list(range(1, 100))) + self.assertEqual(len(b), 99) + alloc = b.__alloc__() + self.assertGreater(alloc, len(b)) + def test_extend(self): orig = b'hello' a = bytearray(orig) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray + object now always allocates place for trailing null byte and it's buffer now + is always null-terminated. + - Issue #24115: Update uses of PyObject_IsTrue(), PyObject_Not(), PyObject_IsInstance(), PyObject_RichCompareBool() and _PyDict_Contains() to check for and handle errors correctly. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -854,8 +854,10 @@ goto error; /* Append the byte */ - if (Py_SIZE(self) < self->ob_alloc) + if (Py_SIZE(self) + 1 < self->ob_alloc) { Py_SIZE(self)++; + PyByteArray_AS_STRING(self)[Py_SIZE(self)] = '\0'; + } else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) goto error; PyByteArray_AS_STRING(self)[Py_SIZE(self)-1] = value; -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

Daily reference leaks (f2f5d1c928eb): sum=-11
by solipsis＠pitrou.net June 29, 2015

June 29, 2015

results for f2f5d1c928eb on branch "default" -------------------------------------------- test_collections leaked [-12, 0, 0] references, sum=-12 test_collections leaked [-6, 2, 0] memory blocks, sum=-4 test_functools leaked [0, 2, 2] memory blocks, sum=4 test_smtplib leaked [0, 1, 0] references, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogRZu6Nf', '--timeout', '7200']

1 0

cpython (2.7): Issue #20387: Backport test from Python 3.4
by jason.coombs June 28, 2015

June 28, 2015

https://hg.python.org/cpython/rev/524a0e755797 changeset: 96715:524a0e755797 branch: 2.7 parent: 96702:1f6c096ee772 user: Jason R. Coombs <jaraco(a)jaraco.com> date: Sun Jun 28 13:03:26 2015 -0400 summary: Issue #20387: Backport test from Python 3.4 files: Lib/test/test_tokenize.py | 20 +++++++++++++++++++- 1 files changed, 19 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py … [View More]

1 0