Mailman 3 December 2015 - Python-checkins

test: test commit to hgtest repo.
by senthil.kumaran 31 Dec '15

31 Dec '15

https://hg.python.org/test/rev/09a6bd753c5d changeset: 228:09a6bd753c5d user: Senthil Kumaran date: Thu Dec 31 07:03:18 2015 -0800 summary: test commit to hgtest repo. testing old rsa key. files: quote | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/quote b/quote --- a/quote +++ b/quote @@ -9,3 +9,7 @@ "Little else matters than to write good code." -- Karl Lehenbauer + +There are only two kinds of languages: the ones people complain about and the ones nobody uses +— Bjarne Stroustrup + -- Repository URL: https://hg.python.org/test

1 0

cpython (2.7): Issue #25961: Fixed compilation error and a leak in type constructor.
by serhiy.storchaka 31 Dec '15

31 Dec '15

https://hg.python.org/cpython/rev/57fea6f75ac2 changeset: 99729:57fea6f75ac2 branch: 2.7 parent: 99726:29cc6b2f9d28 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 31 12:03:14 2015 +0200 summary: Issue #25961: Fixed compilation error and a leak in type constructor. files: Objects/typeobject.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2342,12 +2342,17 @@ type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; type->tp_name = PyString_AS_STRING(name); - if (!type->tp_name) - goto error; + if (!type->tp_name) { + Py_DECREF(bases); + Py_DECREF(type); + return NULL; + } if (strlen(type->tp_name) != (size_t)PyString_GET_SIZE(name)) { PyErr_SetString(PyExc_ValueError, "type name must not contain null characters"); - goto error; + Py_DECREF(bases); + Py_DECREF(type); + return NULL; } /* Set tp_base and tp_bases */ @@ -2369,8 +2374,10 @@ tmp = PyDict_GetItemString(tmp, "__name__"); if (tmp != NULL) { if (PyDict_SetItemString(dict, "__module__", - tmp) < 0) + tmp) < 0) { + Py_DECREF(type); return NULL; + } } } } -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (1ab7bcd4e176): sum=2
by solipsis＠pitrou.net 31 Dec '15

31 Dec '15

results for 1ab7bcd4e176 on branch "default" -------------------------------------------- test_collections leaked [0, 0, -2] references, sum=-2 test_functools leaked [0, 2, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogexQhcM', '--timeout', '7200']

1 0

cpython (merge 3.5 -> default): Issue #25961: Disallowed null characters in the type name.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/1ab7bcd4e176 changeset: 99728:1ab7bcd4e176 parent: 99725:965f0bddac6d parent: 99727:d2417971c934 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 21:41:53 2015 +0200 summary: Issue #25961: Disallowed null characters in the type name. Simplified testing for null characters in __name__ setter. files: Misc/NEWS | 2 + Objects/typeobject.c | 34 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #25961: Disallowed null characters in the type name. + - Issue #25973: Fix segfault when an invalid nonlocal statement binds a name starting with two underscores. diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -401,8 +401,8 @@ static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { - char *tp_name; - PyObject *tmp; + const char *tp_name; + Py_ssize_t name_size; if (!check_set_special_type_attr(type, value, "__name__")) return -1; @@ -413,21 +413,14 @@ return -1; } - /* Check absence of null characters */ - tmp = PyUnicode_FromStringAndSize("\0", 1); - if (tmp == NULL) - return -1; - if (PyUnicode_Contains(value, tmp) != 0) { - Py_DECREF(tmp); - PyErr_Format(PyExc_ValueError, - "__name__ must not contain null bytes"); - return -1; - } - Py_DECREF(tmp); - - tp_name = _PyUnicode_AsString(value); + tp_name = PyUnicode_AsUTF8AndSize(value, &name_size); if (tp_name == NULL) return -1; + if (strlen(tp_name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); + return -1; + } type->tp_name = tp_name; Py_INCREF(value); @@ -2284,8 +2277,8 @@ PyTypeObject *type = NULL, *base, *tmptype, *winner; PyHeapTypeObject *et; PyMemberDef *mp; - Py_ssize_t i, nbases, nslots, slotoffset, add_dict, add_weak; - int j, may_add_dict, may_add_weak; + Py_ssize_t i, nbases, nslots, slotoffset, name_size; + int j, may_add_dict, may_add_weak, add_dict, add_weak; _Py_IDENTIFIER(__qualname__); _Py_IDENTIFIER(__slots__); @@ -2508,9 +2501,14 @@ type->tp_as_sequence = &et->as_sequence; type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - type->tp_name = _PyUnicode_AsString(name); + type->tp_name = PyUnicode_AsUTF8AndSize(name, &name_size); if (!type->tp_name) goto error; + if (strlen(type->tp_name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); + goto error; + } /* Set tp_base and tp_bases */ type->tp_bases = bases; -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #25961: Disallowed null characters in the type name.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/29cc6b2f9d28 changeset: 99726:29cc6b2f9d28 branch: 2.7 parent: 99723:32806d3f9bd2 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 21:39:21 2015 +0200 summary: Issue #25961: Disallowed null characters in the type name. files: Misc/NEWS | 2 ++ Objects/typeobject.c | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #25961: Disallowed null characters in the type name. + - Issue #22995: Instances of extension types with a state that aren't subclasses of list or dict and haven't implemented any pickle-related methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -244,8 +244,8 @@ } if (strlen(PyString_AS_STRING(value)) != (size_t)PyString_GET_SIZE(value)) { - PyErr_Format(PyExc_ValueError, - "__name__ must not contain null bytes"); + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); return -1; } @@ -2071,8 +2071,8 @@ PyTypeObject *type, *base, *tmptype, *winner; PyHeapTypeObject *et; PyMemberDef *mp; - Py_ssize_t i, nbases, nslots, slotoffset, add_dict, add_weak; - int j, may_add_dict, may_add_weak; + Py_ssize_t i, nbases, nslots, slotoffset; + int j, may_add_dict, may_add_weak, add_dict, add_weak; assert(args != NULL && PyTuple_Check(args)); assert(kwds == NULL || PyDict_Check(kwds)); @@ -2342,6 +2342,13 @@ type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; type->tp_name = PyString_AS_STRING(name); + if (!type->tp_name) + goto error; + if (strlen(type->tp_name) != (size_t)PyString_GET_SIZE(name)) { + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); + goto error; + } /* Set tp_base and tp_bases */ type->tp_bases = bases; -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.5): Issue #25961: Disallowed null characters in the type name.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/d2417971c934 changeset: 99727:d2417971c934 branch: 3.5 parent: 99724:b35212d6d4ce user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 21:40:49 2015 +0200 summary: Issue #25961: Disallowed null characters in the type name. Simplified testing for null characters in __name__ setter. files: Misc/NEWS | 2 + Objects/typeobject.c | 49 ++++++++++++------------------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #25961: Disallowed null characters in the type name. + - Issue #25973: Fix segfault when an invalid nonlocal statement binds a name starting with two underscores. diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -401,9 +401,8 @@ static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { - PyHeapTypeObject* et; - char *tp_name; - PyObject *tmp; + const char *tp_name; + Py_ssize_t name_size; if (!check_set_special_type_attr(type, value, "__name__")) return -1; @@ -414,33 +413,18 @@ return -1; } - /* Check absence of null characters */ - tmp = PyUnicode_FromStringAndSize("\0", 1); - if (tmp == NULL) - return -1; - if (PyUnicode_Contains(value, tmp) != 0) { - Py_DECREF(tmp); - PyErr_Format(PyExc_ValueError, - "__name__ must not contain null bytes"); - return -1; - } - Py_DECREF(tmp); - - tp_name = _PyUnicode_AsString(value); + tp_name = PyUnicode_AsUTF8AndSize(value, &name_size); if (tp_name == NULL) return -1; - - et = (PyHeapTypeObject*)type; - + if (strlen(tp_name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); + return -1; + } + + type->tp_name = tp_name; Py_INCREF(value); - - /* Wait until et is a sane state before Py_DECREF'ing the old et->ht_name - value. (Bug #16447.) */ - tmp = et->ht_name; - et->ht_name = value; - - type->tp_name = tp_name; - Py_DECREF(tmp); + Py_SETREF(((PyHeapTypeObject*)type)->ht_name, value); return 0; } @@ -2285,8 +2269,8 @@ PyTypeObject *type = NULL, *base, *tmptype, *winner; PyHeapTypeObject *et; PyMemberDef *mp; - Py_ssize_t i, nbases, nslots, slotoffset, add_dict, add_weak; - int j, may_add_dict, may_add_weak; + Py_ssize_t i, nbases, nslots, slotoffset, name_size; + int j, may_add_dict, may_add_weak, add_dict, add_weak; _Py_IDENTIFIER(__qualname__); _Py_IDENTIFIER(__slots__); @@ -2509,9 +2493,14 @@ type->tp_as_sequence = &et->as_sequence; type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - type->tp_name = _PyUnicode_AsString(name); + type->tp_name = PyUnicode_AsUTF8AndSize(name, &name_size); if (!type->tp_name) goto error; + if (strlen(type->tp_name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, + "type name must not contain null characters"); + goto error; + } /* Set tp_base and tp_bases */ type->tp_bases = bases; -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Make catched exceptions more specific and correct a comment.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/32806d3f9bd2 changeset: 99723:32806d3f9bd2 branch: 2.7 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 20:59:32 2015 +0200 summary: Make catched exceptions more specific and correct a comment. files: Lib/test/pickletester.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1373,15 +1373,16 @@ # 5th item is not an iterator return dict, (), None, None, [] - # Protocol 0 is less strict and also accept iterables. + # Protocol 0 in Python implementation is less strict and also accepts + # iterables. for proto in protocols: try: self.dumps(C(), proto) - except (AttributeError, pickle.PickleError, cPickle.PickleError): + except (AttributeError, pickle.PicklingError, cPickle.PicklingError): pass try: self.dumps(D(), proto) - except (AttributeError, pickle.PickleError, cPickle.PickleError): + except (AttributeError, pickle.PicklingError, cPickle.PicklingError): pass def test_many_puts_and_gets(self): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.5): Make catched exception more specific and correct a comment.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/b35212d6d4ce changeset: 99724:b35212d6d4ce branch: 3.5 parent: 99720:4679b841e5ef user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 21:00:08 2015 +0200 summary: Make catched exception more specific and correct a comment. files: Lib/test/pickletester.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1944,15 +1944,15 @@ # 5th item is not an iterator return dict, (), None, None, [] - # Protocol 0 is less strict and also accept iterables. + # Python implementation is less strict and also accepts iterables. for proto in protocols: try: self.dumps(C(), proto) - except (pickle.PickleError): + except pickle.PicklingError: pass try: self.dumps(D(), proto) - except (pickle.PickleError): + except pickle.PicklingError: pass def test_many_puts_and_gets(self): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.5 -> default): Make catched exception more specific and correct a comment.
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/965f0bddac6d changeset: 99725:965f0bddac6d parent: 99721:0155eefa8162 parent: 99724:b35212d6d4ce user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 21:00:29 2015 +0200 summary: Make catched exception more specific and correct a comment. files: Lib/test/pickletester.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1942,15 +1942,15 @@ # 5th item is not an iterator return dict, (), None, None, [] - # Protocol 0 is less strict and also accept iterables. + # Python implementation is less strict and also accepts iterables. for proto in protocols: try: self.dumps(C(), proto) - except (pickle.PickleError): + except pickle.PicklingError: pass try: self.dumps(D(), proto) - except (pickle.PickleError): + except pickle.PicklingError: pass def test_many_puts_and_gets(self): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #22995: Instances of extension types with a state that aren't
by serhiy.storchaka 30 Dec '15

30 Dec '15

https://hg.python.org/cpython/rev/92172d7372dd changeset: 99722:92172d7372dd branch: 2.7 parent: 99688:11670e4be1a9 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Wed Dec 30 20:43:29 2015 +0200 summary: Issue #22995: Instances of extension types with a state that aren't subclasses of list or dict and haven't implemented any pickle-related methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, or __getstate__), can no longer be pickled. Including memoryview. files: Lib/test/test_buffer.py | 13 +++++++ Lib/test/test_csv.py | 13 +++++++ Lib/test/test_memoryview.py | 15 +++++++++ Misc/NEWS | 5 +++ Objects/typeobject.c | 41 ++++++++++++++++++++---- 5 files changed, 79 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4,6 +4,8 @@ """ +import copy +import pickle import sys import unittest from test import test_support @@ -35,6 +37,17 @@ buf = buffer(data, sys.maxsize, sys.maxsize) self.assertEqual(buf[:4096], "") + def test_copy(self): + buf = buffer(b'abc') + with self.assertRaises(TypeError): + copy.copy(buf) + + def test_pickle(self): + buf = buffer(b'abc') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(buf, proto) + def test_main(): with test_support.check_py3k_warnings(("buffer.. not supported", diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -2,6 +2,7 @@ # Copyright (C) 2001,2002 Python Software Foundation # csv package unit tests +import copy import sys import os import unittest @@ -10,6 +11,7 @@ import csv import gc import io +import pickle from test import test_support class Test_Csv(unittest.TestCase): @@ -466,6 +468,17 @@ self.assertRaises(TypeError, csv.reader, [], quoting = -1) self.assertRaises(TypeError, csv.reader, [], quoting = 100) + def test_copy(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + self.assertRaises(TypeError, copy.copy, dialect) + + def test_pickle(self): + for name in csv.list_dialects(): + dialect = csv.get_dialect(name) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, dialect, proto) + class TestCsvBase(unittest.TestCase): def readerAssertEqual(self, input, expected_result): fd, name = tempfile.mkstemp() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -10,6 +10,8 @@ import array from test import test_support import io +import copy +import pickle class AbstractMemoryTests: @@ -354,6 +356,19 @@ #pass +class OtherTest(unittest.TestCase): + def test_copy(self): + m = memoryview(b'abc') + with self.assertRaises(TypeError): + copy.copy(m) + + def test_pickle(self): + m = memoryview(b'abc') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(m, proto) + + def test_main(): test_support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Core and Builtins ----------------- +- Issue #22995: Instances of extension types with a state that aren't + subclasses of list or dict and haven't implemented any pickle-related + methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, + or __getstate__), can no longer be pickled. Including memoryview. + - Issue #20440: Massive replacing unsafe attribute setting code with special macro Py_SETREF. diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3207,6 +3207,7 @@ PyObject *slots = NULL, *listitems = NULL, *dictitems = NULL; PyObject *copyreg = NULL, *newobj = NULL, *res = NULL; Py_ssize_t i, n; + int required_state = 0; cls = PyObject_GetAttrString(obj, "__class__"); if (cls == NULL) @@ -3214,7 +3215,7 @@ if (PyType_Check(cls) && ((PyTypeObject *)cls)->tp_new == NULL) { PyErr_Format(PyExc_TypeError, - "can't pickle %s objects", + "can't pickle %.200s objects", ((PyTypeObject *)cls)->tp_name); return NULL; } @@ -3223,7 +3224,9 @@ if (getnewargs != NULL) { args = PyObject_CallObject(getnewargs, NULL); Py_DECREF(getnewargs); - if (args != NULL && !PyTuple_Check(args)) { + if (args == NULL) + goto end; + if (!PyTuple_Check(args)) { PyErr_Format(PyExc_TypeError, "__getnewargs__ should return a tuple, " "not '%.200s'", Py_TYPE(args)->tp_name); @@ -3232,10 +3235,8 @@ } else { PyErr_Clear(); - args = PyTuple_New(0); - } - if (args == NULL) - goto end; + required_state = !PyList_Check(obj) && !PyDict_Check(obj); + } getstate = PyObject_GetAttrString(obj, "__getstate__"); if (getstate != NULL) { @@ -3246,6 +3247,14 @@ } else { PyErr_Clear(); + + if (required_state && obj->ob_type->tp_itemsize) { + PyErr_Format(PyExc_TypeError, + "can't pickle %.200s objects", + Py_TYPE(obj)->tp_name); + goto end; + } + state = PyObject_GetAttrString(obj, "__dict__"); if (state == NULL) { PyErr_Clear(); @@ -3255,8 +3264,24 @@ names = slotnames(cls); if (names == NULL) goto end; + assert(names == Py_None || PyList_Check(names)); + if (required_state) { + Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize; + if (obj->ob_type->tp_dictoffset) + basicsize += sizeof(PyObject *); + if (obj->ob_type->tp_weaklistoffset) + basicsize += sizeof(PyObject *); + if (names != Py_None) + basicsize += sizeof(PyObject *) * Py_SIZE(names); + if (obj->ob_type->tp_basicsize > basicsize) { + PyErr_Format(PyExc_TypeError, + "can't pickle %.200s objects", + Py_TYPE(obj)->tp_name); + goto end; + } + } + if (names != Py_None) { - assert(PyList_Check(names)); slots = PyDict_New(); if (slots == NULL) goto end; @@ -3318,7 +3343,7 @@ if (newobj == NULL) goto end; - n = PyTuple_GET_SIZE(args); + n = args ? PyTuple_GET_SIZE(args) : 0; args2 = PyTuple_New(n+1); if (args2 == NULL) goto end; -- Repository URL: https://hg.python.org/cpython

1 0