Mailman 3 November 2015 - Python-checkins

cpython (2.7): Issue #25761: Added more test cases for testing unpickling broken data.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/c7e7d77ef8bf changeset: 99383:c7e7d77ef8bf branch: 2.7 parent: 99379:7841e9b614eb user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 13:12:40 2015 +0200 summary: Issue #25761: Added more test cases for testing unpickling broken data. Output raised exception at verbose level 2 (-vv). files: Lib/test/pickletester.py | 227 +++++++++++++++++--------- Lib/test/test_cpickle.py | 6 + Lib/test/test_pickle.py | 13 + 3 files changed, 170 insertions(+), 76 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -6,7 +6,9 @@ import cStringIO import pickletools import copy_reg +import sys +from test import test_support as support from test.test_support import TestFailed, verbose, have_unicode, TESTFN try: from test.test_support import _2G, _1M, precisionbigmemtest @@ -473,6 +475,17 @@ self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) + def check_unpickling_error(self, errors, data): + with self.assertRaises(errors): + try: + self.loads(data) + except: + if support.verbose > 1: + exc = sys.exc_info()[1] + print('%-32r - %s: %s' % + (data, exc.__class__.__name__, exc)) + raise + def test_load_from_canned_string(self): expected = self._testdata for canned in DATA0, DATA1, DATA2: @@ -480,7 +493,7 @@ self.assert_is_copy(expected, got) def test_garyp(self): - self.assertRaises(self.error, self.loads, 'garyp') + self.check_unpickling_error(self.error, 'garyp') def test_maxint64(self): maxint64 = (1L << 63) - 1 @@ -490,7 +503,7 @@ # Try too with a bogus literal. data = 'I' + str(maxint64) + 'JUNK\n.' - self.assertRaises(ValueError, self.loads, data) + self.check_unpickling_error(ValueError, data) def test_insecure_strings(self): insecure = ["abc", "2 + 2", # not quoted @@ -511,7 +524,7 @@ ] for s in insecure: buf = "S" + s + "\n." - self.assertRaises(ValueError, self.loads, buf) + self.check_unpickling_error(ValueError, buf) def test_correctly_quoted_string(self): goodpickles = [("S''\n.", ''), @@ -578,11 +591,6 @@ 'q\x00oq\x01}q\x02b.').replace('X', xname) self.assert_is_copy(X(*args), self.loads(pickle2)) - def test_pop_empty_stack(self): - # Test issue7455 - s = '0' - self.assertRaises((cPickle.UnpicklingError, IndexError), self.loads, s) - def test_load_str(self): # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) self.assertEqual(self.loads("S'a\\x00\\xa0'\n."), 'a\x00\xa0') @@ -607,8 +615,8 @@ self.assertIs(self.loads('I00\n.'), False) def test_misc_get(self): - self.assertRaises(self.error, self.loads, 'g0\np0\n') - self.assertRaises(self.error, self.loads, 'h\x00q\x00') + self.check_unpickling_error(self.error, 'g0\np0\n') + self.check_unpickling_error(self.error, 'h\x00q\x00') def test_get(self): pickled = '((lp100000\ng100000\nt.' @@ -636,74 +644,145 @@ def test_bad_stack(self): badpickles = [ - '0.', # POP - '1.', # POP_MARK - '2.', # DUP - # '(2.', # PyUnpickler doesn't raise - 'R.', # REDUCE - ')R.', - 'a.', # APPEND - 'Na.', - 'b.', # BUILD - 'Nb.', - 'd.', # DICT - 'e.', # APPENDS - # '(e.', # PyUnpickler raises AttributeError - 'i__builtin__\nlist\n.', # INST - 'l.', # LIST - 'o.', # OBJ - '(o.', - 'p1\n.', # PUT - 'q\x00.', # BINPUT - 'r\x00\x00\x00\x00.', # LONG_BINPUT - 's.', # SETITEM - 'Ns.', - 'NNs.', - 't.', # TUPLE - 'u.', # SETITEMS - '(u.', - '}(Nu.', - '\x81.', # NEWOBJ - ')\x81.', - '\x85.', # TUPLE1 - '\x86.', # TUPLE2 - 'N\x86.', - '\x87.', # TUPLE3 - 'N\x87.', - 'NN\x87.', + '.', # STOP + '0', # POP + '1', # POP_MARK + '2', # DUP + # '(2', # PyUnpickler doesn't raise + 'R', # REDUCE + ')R', + 'a', # APPEND + 'Na', + 'b', # BUILD + 'Nb', + 'd', # DICT + 'e', # APPENDS + # '(e', # PyUnpickler raises AttributeError + 'i__builtin__\nlist\n', # INST + 'l', # LIST + 'o', # OBJ + '(o', + 'p1\n', # PUT + 'q\x00', # BINPUT + 'r\x00\x00\x00\x00', # LONG_BINPUT + 's', # SETITEM + 'Ns', + 'NNs', + 't', # TUPLE + 'u', # SETITEMS + # '(u', # PyUnpickler doesn't raise + '}(Nu', + '\x81', # NEWOBJ + ')\x81', + '\x85', # TUPLE1 + '\x86', # TUPLE2 + 'N\x86', + '\x87', # TUPLE3 + 'N\x87', + 'NN\x87', ] for p in badpickles: - try: - self.assertRaises(self.bad_stack_errors, self.loads, p) - except: - print '***', repr(p) - raise + self.check_unpickling_error(self.bad_stack_errors, p) def test_bad_mark(self): badpickles = [ - 'c__builtin__\nlist\n)(R.', # REDUCE - 'c__builtin__\nlist\n()R.', - ']N(a.', # APPEND - 'cexceptions\nValueError\n)R}(b.', # BUILD - 'cexceptions\nValueError\n)R(}b.', - '(Nd.', # DICT - '}NN(s.', # SETITEM - '}N(Ns.', - 'c__builtin__\nlist\n)(\x81.', # NEWOBJ - 'c__builtin__\nlist\n()\x81.', - 'N(\x85.', # TUPLE1 - 'NN(\x86.', # TUPLE2 - 'N(N\x86.', - 'NNN(\x87.', # TUPLE3 - 'NN(N\x87.', - 'N(NN\x87.', + # 'N(.', # STOP + 'N(2', # DUP + 'c__builtin__\nlist\n)(R', # REDUCE + 'c__builtin__\nlist\n()R', + ']N(a', # APPEND + # BUILD + 'c__builtin__\nValueError\n)R}(b', + 'c__builtin__\nValueError\n)R(}b', + '(Nd', # DICT + 'N(p1\n', # PUT + 'N(q\x00', # BINPUT + 'N(r\x00\x00\x00\x00', # LONG_BINPUT + '}NN(s', # SETITEM + '}N(Ns', + '}(NNs', + '}((u', # SETITEMS + # NEWOBJ + 'c__builtin__\nlist\n)(\x81', + 'c__builtin__\nlist\n()\x81', + 'N(\x85', # TUPLE1 + 'NN(\x86', # TUPLE2 + 'N(N\x86', + 'NNN(\x87', # TUPLE3 + 'NN(N\x87', + 'N(NN\x87', ] for p in badpickles: - try: - self.loads(p) - except (IndexError, AttributeError, TypeError, - pickle.UnpicklingError): - pass + self.check_unpickling_error(self.bad_mark_errors, p) + + def test_truncated_data(self): + self.check_unpickling_error(EOFError, b'') + self.check_unpickling_error(EOFError, b'N') + badpickles = [ + 'F', # FLOAT + 'F0.0', + 'F0.00', + 'G', # BINFLOAT + 'G\x00\x00\x00\x00\x00\x00\x00', + 'I', # INT + 'I0', + 'J', # BININT + 'J\x00\x00\x00', + 'K', # BININT1 + 'L', # LONG + 'L0', + 'L10', + 'L0L', + 'L10L', + 'M', # BININT2 + 'M\x00', + # 'P', # PERSID + # 'Pabc', + 'S', # STRING + b"S'abc'", + 'T', # BINSTRING + 'T\x03\x00\x00', + 'T\x03\x00\x00\x00', + 'T\x03\x00\x00\x00ab', + 'U', # SHORT_BINSTRING + 'U\x03', + 'U\x03ab', + 'V', # UNICODE + 'Vabc', + 'X', # BINUNICODE + 'X\x03\x00\x00', + 'X\x03\x00\x00\x00', + 'X\x03\x00\x00\x00ab', + '(c', # GLOBAL + '(c__builtin__', + '(c__builtin__\n', + '(c__builtin__\nlist', + 'Ng', # GET + 'Ng0', + '(i', # INST + '(i__builtin__', + '(i__builtin__\n', + '(i__builtin__\nlist', + 'Nh', # BINGET + 'Nj', # LONG_BINGET + 'Nj\x00\x00\x00', + 'Np', # PUT + 'Np0', + 'Nq', # BINPUT + 'Nr', # LONG_BINPUT + 'Nr\x00\x00\x00', + '\x80', # PROTO + '\x82', # EXT1 + '\x83', # EXT2 + '\x84\x01', + '\x84', # EXT4 + '\x84\x01\x00\x00', + '\x8a', # LONG1 + '\x8b', # LONG4 + '\x8b\x00\x00\x00', + ] + for p in badpickles: + self.check_unpickling_error(self.truncated_errors, p) class AbstractPickleTests(unittest.TestCase): @@ -1467,11 +1546,7 @@ # Test issue4298 s = '\x58\0\0\0\x54' self.assertRaises(EOFError, self.module.loads, s) - # Test issue7455 - s = '0' - # XXX Why doesn't pickle raise UnpicklingError? - self.assertRaises((IndexError, cPickle.UnpicklingError), - self.module.loads, s) + class AbstractPersistentPicklerTests(unittest.TestCase): diff --git a/Lib/test/test_cpickle.py b/Lib/test/test_cpickle.py --- a/Lib/test/test_cpickle.py +++ b/Lib/test/test_cpickle.py @@ -52,6 +52,9 @@ error = cPickle.BadPickleGet module = cPickle bad_stack_errors = (cPickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (cPickle.UnpicklingError, EOFError, + AttributeError, ValueError) class cPickleUnpicklerTests(AbstractUnpickleTests): @@ -65,6 +68,9 @@ error = cPickle.BadPickleGet bad_stack_errors = (cPickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (cPickle.UnpicklingError, EOFError, + AttributeError, ValueError) class cStringIOCUnpicklerTests(cStringIOMixin, cPickleUnpicklerTests): pass diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -1,4 +1,5 @@ import pickle +import struct from cStringIO import StringIO from test import test_support @@ -24,11 +25,23 @@ module = pickle error = KeyError bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError, + TypeError, KeyError) class UnpicklerTests(AbstractUnpickleTests): error = KeyError bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError, + TypeError, KeyError) def loads(self, buf): f = StringIO(buf) -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #25761: Added more test cases for testing unpickling broken data.
by serhiy.storchaka Nov. 29, 2015

Nov. 29, 2015

https://hg.python.org/cpython/rev/d47e5b162072 changeset: 99382:d47e5b162072 branch: 3.4 parent: 99375:aa304ad32292 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 29 13:12:10 2015 +0200 summary: Issue #25761: Added more test cases for testing unpickling broken data. Output raised exception at verbose level 2 (-vv). files: Lib/test/pickletester.py | 290 ++++++++++++++++++-------- Lib/test/test_pickle.py | 13 + 2 files changed, 210 insertions(+), 93 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -11,8 +11,9 @@ import weakref from http.cookies import SimpleCookie +from test import support from test.support import ( - TestFailed, TESTFN, run_with_locale, no_tracing, captured_stdout, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, ) @@ -679,6 +680,17 @@ self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) + def check_unpickling_error(self, errors, data): + with self.subTest(data=data), \ + self.assertRaises(errors): + try: + self.loads(data) + except BaseException as exc: + if support.verbose > 1: + print('%-32r - %s: %s' % + (data, exc.__class__.__name__, exc)) + raise + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -759,12 +771,7 @@ # Try too with a bogus literal. data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' - self.assertRaises(ValueError, self.loads, data) - - def test_pop_empty_stack(self): - # Test issue7455 - s = b'0' - self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s) + self.check_unpickling_error(ValueError, data) def test_unpickle_from_2x(self): # Unpickle non-trivial data from Python 2.x. @@ -849,22 +856,22 @@ def test_negative_32b_binbytes(self): # On 32-bit builds, a BINBYTES of 2**31 or more is refused dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_negative_32b_binunicode(self): # On 32-bit builds, a BINUNICODE of 2**31 or more is refused dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_short_binunicode(self): dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' self.assertEqual(self.loads(dumped), '\u20ac\x00') def test_misc_get(self): - self.assertRaises(KeyError, self.loads, b'g0\np0') + self.check_unpickling_error(KeyError, b'g0\np0') self.assert_is_copy([(100,), (100,)], self.loads(b'((Kdtp0\nh\x00l.))')) @@ -879,14 +886,14 @@ @requires_32b def test_large_32b_binbytes8(self): dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_large_32b_binunicode8(self): dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_get(self): pickled = b'((lp100000\ng100000\nt.' @@ -915,16 +922,16 @@ def test_negative_put(self): # Issue #12847 dumped = b'Va\np-1\n.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) @requires_32b def test_negative_32b_binput(self): # Issue #12847 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) def test_badly_escaped_string(self): - self.assertRaises(ValueError, self.loads, b"S'\\'\n.") + self.check_unpickling_error(ValueError, b"S'\\'\n.") def test_badly_quoted_string(self): # Issue #17710 @@ -942,7 +949,7 @@ b'S\n.', b'S.'] for p in badpickles: - self.assertRaises(pickle.UnpicklingError, self.loads, p) + self.check_unpickling_error(pickle.UnpicklingError, p) def test_correctly_quoted_string(self): goodpickles = [(b"S''\n.", ''), @@ -989,86 +996,183 @@ def test_bad_stack(self): badpickles = [ - b'0.', # POP - b'1.', # POP_MARK - b'2.', # DUP - # b'(2.', # PyUnpickler doesn't raise - b'R.', # REDUCE - b')R.', - b'a.', # APPEND - b'Na.', - b'b.', # BUILD - b'Nb.', - b'd.', # DICT - b'e.', # APPENDS - # b'(e.', # PyUnpickler raises AttributeError - b'ibuiltins\nlist\n.', # INST - b'l.', # LIST - b'o.', # OBJ - b'(o.', - b'p1\n.', # PUT - b'q\x00.', # BINPUT - b'r\x00\x00\x00\x00.', # LONG_BINPUT - b's.', # SETITEM - b'Ns.', - b'NNs.', - b't.', # TUPLE - b'u.', # SETITEMS - b'(u.', - b'}(Nu.', - b'\x81.', # NEWOBJ - b')\x81.', - b'\x85.', # TUPLE1 - b'\x86.', # TUPLE2 - b'N\x86.', - b'\x87.', # TUPLE3 - b'N\x87.', - b'NN\x87.', - b'\x90.', # ADDITEMS - # b'(\x90.', # PyUnpickler raises AttributeError - b'\x91.', # FROZENSET - b'\x92.', # NEWOBJ_EX - b')}\x92.', - b'\x93.', # STACK_GLOBAL - b'Vlist\n\x93.', - b'\x94.', # MEMOIZE + b'.', # STOP + b'0', # POP + b'1', # POP_MARK + b'2', # DUP + # b'(2', # PyUnpickler doesn't raise + b'R', # REDUCE + b')R', + b'a', # APPEND + b'Na', + b'b', # BUILD + b'Nb', + b'd', # DICT + b'e', # APPENDS + # b'(e', # PyUnpickler raises AttributeError + b'ibuiltins\nlist\n', # INST + b'l', # LIST + b'o', # OBJ + b'(o', + b'p1\n', # PUT + b'q\x00', # BINPUT + b'r\x00\x00\x00\x00', # LONG_BINPUT + b's', # SETITEM + b'Ns', + b'NNs', + b't', # TUPLE + b'u', # SETITEMS + # b'(u', # PyUnpickler doesn't raise + b'}(Nu', + b'\x81', # NEWOBJ + b')\x81', + b'\x85', # TUPLE1 + b'\x86', # TUPLE2 + b'N\x86', + b'\x87', # TUPLE3 + b'N\x87', + b'NN\x87', + b'\x90', # ADDITEMS + # b'(\x90', # PyUnpickler raises AttributeError + b'\x91', # FROZENSET + b'\x92', # NEWOBJ_EX + b')}\x92', + b'\x93', # STACK_GLOBAL + b'Vlist\n\x93', + b'\x94', # MEMOIZE ] for p in badpickles: - with self.subTest(p): - self.assertRaises(self.bad_stack_errors, self.loads, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_bad_mark(self): badpickles = [ - b'cbuiltins\nlist\n)(R.', # REDUCE - b'cbuiltins\nlist\n()R.', - b']N(a.', # APPEND - b'cbuiltins\nValueError\n)R}(b.', # BUILD - b'cbuiltins\nValueError\n)R(}b.', - b'(Nd.', # DICT - b'}NN(s.', # SETITEM - b'}N(Ns.', - b'cbuiltins\nlist\n)(\x81.', # NEWOBJ - b'cbuiltins\nlist\n()\x81.', - b'N(\x85.', # TUPLE1 - b'NN(\x86.', # TUPLE2 - b'N(N\x86.', - b'NNN(\x87.', # TUPLE3 - b'NN(N\x87.', - b'N(NN\x87.', - b'cbuiltins\nlist\n)}(\x92.', # NEWOBJ_EX - b'cbuiltins\nlist\n)(}\x92.', - b'cbuiltins\nlist\n()}\x92.', - b'Vbuiltins\n(Vlist\n\x93.', # STACK_GLOBAL - b'Vbuiltins\nVlist\n(\x93.', + # b'N(.', # STOP + b'N(2', # DUP + b'cbuiltins\nlist\n)(R', # REDUCE + b'cbuiltins\nlist\n()R', + b']N(a', # APPEND + # BUILD + b'cbuiltins\nValueError\n)R}(b', + b'cbuiltins\nValueError\n)R(}b', + b'(Nd', # DICT + b'N(p1\n', # PUT + b'N(q\x00', # BINPUT + b'N(r\x00\x00\x00\x00', # LONG_BINPUT + b'}NN(s', # SETITEM + b'}N(Ns', + b'}(NNs', + b'}((u', # SETITEMS + b'cbuiltins\nlist\n)(\x81', # NEWOBJ + b'cbuiltins\nlist\n()\x81', + b'N(\x85', # TUPLE1 + b'NN(\x86', # TUPLE2 + b'N(N\x86', + b'NNN(\x87', # TUPLE3 + b'NN(N\x87', + b'N(NN\x87', + b']((\x90', # ADDITEMS + # NEWOBJ_EX + b'cbuiltins\nlist\n)}(\x92', + b'cbuiltins\nlist\n)(}\x92', + b'cbuiltins\nlist\n()}\x92', + # STACK_GLOBAL + b'Vbuiltins\n(Vlist\n\x93', + b'Vbuiltins\nVlist\n(\x93', + b'N(\x94', # MEMOIZE ] for p in badpickles: - # PyUnpickler prints reduce errors to stdout - with self.subTest(p), captured_stdout(): - try: - self.loads(p) - except (IndexError, AttributeError, TypeError, - pickle.UnpicklingError): - pass + self.check_unpickling_error(self.bad_mark_errors, p) + + def test_truncated_data(self): + self.check_unpickling_error(EOFError, b'') + self.check_unpickling_error(EOFError, b'N') + badpickles = [ + b'B', # BINBYTES + b'B\x03\x00\x00', + b'B\x03\x00\x00\x00', + b'B\x03\x00\x00\x00ab', + b'C', # SHORT_BINBYTES + b'C\x03', + b'C\x03ab', + b'F', # FLOAT + b'F0.0', + b'F0.00', + b'G', # BINFLOAT + b'G\x00\x00\x00\x00\x00\x00\x00', + b'I', # INT + b'I0', + b'J', # BININT + b'J\x00\x00\x00', + b'K', # BININT1 + b'L', # LONG + b'L0', + b'L10', + b'L0L', + b'L10L', + b'M', # BININT2 + b'M\x00', + # b'P', # PERSID + # b'Pabc', + b'S', # STRING + b"S'abc'", + b'T', # BINSTRING + b'T\x03\x00\x00', + b'T\x03\x00\x00\x00', + b'T\x03\x00\x00\x00ab', + b'U', # SHORT_BINSTRING + b'U\x03', + b'U\x03ab', + b'V', # UNICODE + b'Vabc', + b'X', # BINUNICODE + b'X\x03\x00\x00', + b'X\x03\x00\x00\x00', + b'X\x03\x00\x00\x00ab', + b'(c', # GLOBAL + b'(cbuiltins', + b'(cbuiltins\n', + b'(cbuiltins\nlist', + b'Ng', # GET + b'Ng0', + b'(i', # INST + b'(ibuiltins', + b'(ibuiltins\n', + b'(ibuiltins\nlist', + b'Nh', # BINGET + b'Nj', # LONG_BINGET + b'Nj\x00\x00\x00', + b'Np', # PUT + b'Np0', + b'Nq', # BINPUT + b'Nr', # LONG_BINPUT + b'Nr\x00\x00\x00', + b'\x80', # PROTO + b'\x82', # EXT1 + b'\x83', # EXT2 + b'\x84\x01', + b'\x84', # EXT4 + b'\x84\x01\x00\x00', + b'\x8a', # LONG1 + b'\x8b', # LONG4 + b'\x8b\x00\x00\x00', + b'\x8c', # SHORT_BINUNICODE + b'\x8c\x03', + b'\x8c\x03ab', + b'\x8d', # BINUNICODE8 + b'\x8d\x03\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x8e', # BINBYTES8 + b'\x8e\x03\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x95', # FRAME + b'\x95\x02\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', + ] + for p in badpickles: + self.check_unpickling_error(self.truncated_errors, p) class AbstractPickleTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,6 +33,11 @@ unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def loads(self, buf, **kwds): f = io.BytesIO(buf) @@ -64,6 +69,11 @@ pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) + bad_mark_errors = (pickle.UnpicklingError, IndexError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def dumps(self, arg, protocol=None): return pickle.dumps(arg, protocol) @@ -122,6 +132,9 @@ class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError) class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (2e889344436e): sum=8
by solipsis＠pitrou.net Nov. 29, 2015

Nov. 29, 2015

results for 2e889344436e on branch "default" -------------------------------------------- test_asyncio leaked [3, 0, 0] memory blocks, sum=3 test_functools leaked [0, 2, 2] memory blocks, sum=4 test_smtplib leaked [0, 1, 0] references, sum=1 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogsO_r6H', '--timeout', '7200']

1 0

cpython: Issue #25754: Allow test_rlcompleter to be run multiple times
by martin.panter Nov. 28, 2015

Nov. 28, 2015

https://hg.python.org/cpython/rev/2e889344436e changeset: 99381:2e889344436e user: Martin Panter <vadmium+py(a)gmail.com> date: Sat Nov 28 22:38:24 2015 +0000 summary: Issue #25754: Allow test_rlcompleter to be run multiple times files: Lib/test/test_rlcompleter.py | 13 +++++++------ 1 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import builtins import rlcompleter @@ -72,12 +73,12 @@ self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) - CompleteMe.me = CompleteMe - self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), - ['CompleteMe.me.me.spam']) - self.assertEqual(self.completer.attr_matches('egg.s'), - ['egg.{}('.format(x) for x in dir(str) - if x.startswith('s')]) + with patch.object(CompleteMe, "me", CompleteMe, create=True): + self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), + ['CompleteMe.me.me.spam']) + self.assertEqual(self.completer.attr_matches('egg.s'), + ['egg.{}('.format(x) for x in dir(str) + if x.startswith('s')]) def test_excessive_getattr(self): # Ensure getattr() is invoked no more than once per attribute -- Repository URL: https://hg.python.org/cpython

1 0

cpython: #25485: Add context manager support to Telnet class.
by r.david.murray Nov. 28, 2015

Nov. 28, 2015

https://hg.python.org/cpython/rev/277824f2d133 changeset: 99380:277824f2d133 parent: 99377:9fcfdb53e8af user: R David Murray <rdmurray(a)bitdance.com> date: Sat Nov 28 12:24:52 2015 -0500 summary: #25485: Add context manager support to Telnet class. Patch by Stéphane Wirtel. files: Doc/library/telnetlib.rst | 11 +++++++++++ Doc/whatsnew/3.6.rst | 7 +++++++ Lib/telnetlib.py | 15 ++++++++++----- Lib/test/test_telnetlib.py | 5 +++++ Misc/NEWS | 2 ++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -43,6 +43,17 @@ :exc:`EOFError` when the end of the connection is read, because they can return an empty string for other reasons. See the individual descriptions below. + A :class:`Telnet` object is a context manager and can be used in a + :keyword:`with` statement. When the :keyword:`with` block ends, the + :meth:`close` method is called:: + + >>> from telnetlib import Telnet + >>> with Telnet('localhost', 23) as tn: + ... tn.interact() + ... + + .. versionchanged:: 3.6 Context manager support added + .. seealso:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -125,6 +125,13 @@ an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +telnetlib +--------- + +:class:`~telnetlib.Telnet` is now a context manager (contributed by +Stéphane Wirtel in :issue:`25485`). + + urllib.robotparser ------------------ diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -637,6 +637,12 @@ raise EOFError return (-1, None, text) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + def test(): """Test program for telnetlib. @@ -660,11 +666,10 @@ port = int(portstr) except ValueError: port = socket.getservbyname(portstr, 'tcp') - tn = Telnet() - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - tn.close() + with Telnet() as tn: + tn.set_debuglevel(debuglevel) + tn.open(host, port, timeout=0.5) + tn.interact() if __name__ == '__main__': test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -42,6 +42,11 @@ telnet = telnetlib.Telnet(HOST, self.port) telnet.sock.close() + def testContextManager(self): + with telnetlib.Telnet(HOST, self.port) as tn: + self.assertIsNotNone(tn.get_socket()) + self.assertIsNone(tn.get_socket()) + def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #25485: telnetlib.Telnet is now a context manager. + - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (9fcfdb53e8af): sum=7
by solipsis＠pitrou.net Nov. 28, 2015

Nov. 28, 2015

results for 9fcfdb53e8af on branch "default" -------------------------------------------- test_asyncio leaked [0, 3, 0] memory blocks, sum=3 test_functools leaked [0, 2, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogx_IaGX', '--timeout', '7200']

1 0

cpython (2.7): Closes #25742: locale.setlocale() now accepts a Unicode string for its second
by victor.stinner Nov. 27, 2015

Nov. 27, 2015

https://hg.python.org/cpython/rev/7841e9b614eb changeset: 99379:7841e9b614eb branch: 2.7 user: Victor Stinner <victor.stinner(a)gmail.com> date: Fri Nov 27 23:54:36 2015 +0100 summary: Closes #25742: locale.setlocale() now accepts a Unicode string for its second parameter. files: Lib/locale.py | 6 +++++- Lib/test/test_locale.py | 10 ++++++++++ Misc/NEWS | 3 +++ 3 files changed, 18 insertions(+), 1 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py --- a/Lib/locale.py +++ b/Lib/locale.py @@ -18,6 +18,10 @@ import operator import functools +# keep a copy of the builtin str type, because 'str' name is overriden +# in globals by a function below +_str = str + try: _unicode = unicode except NameError: @@ -573,7 +577,7 @@ category may be given as one of the LC_* values. """ - if locale and type(locale) is not type(""): + if locale and not isinstance(locale, (_str, _unicode)): # convert to string locale = normalize(_build_localename(locale)) return _setlocale(category, locale) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -493,6 +493,16 @@ # longer accept unicode strings. self.assertEqual(locale.normalize(u'en_US'), 'en_US.ISO8859-1') + def test_setlocale_unicode(self): + old_loc = locale.getlocale(locale.LC_ALL) + try: + user_locale = locale.setlocale(locale.LC_ALL, '') + unicode_locale = user_locale.decode('utf-8') + user_locale2 = locale.setlocale(locale.LC_ALL, unicode_locale) + self.assertEqual(user_locale, user_locale2) + finally: + locale.setlocale(locale.LC_ALL, old_loc) + def test_main(): tests = [ diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,6 +20,9 @@ Library ------- +- Issue #25742: :func:`locale.setlocale` now accepts a Unicode string for + its second parameter. + - Issue #10131: Fixed deep copying of minidom documents. Based on patch by Marian Ganisin. -- Repository URL: https://hg.python.org/cpython

1 0

benchmarks: Issue #25719: Deprecate spitfire benchmark.
by zach.ware Nov. 27, 2015

Nov. 27, 2015

https://hg.python.org/benchmarks/rev/75751d838789 changeset: 236:75751d838789 user: Zachary Ware <zachary.ware(a)gmail.com> date: Fri Nov 27 11:38:22 2015 -0600 summary: Issue #25719: Deprecate spitfire benchmark. It makes use of psycho, which does not support 64bit platforms at all and is long dead. Patch by Florin Papa. files: perf.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/perf.py b/perf.py --- a/perf.py +++ b/perf.py @@ -2318,7 +2318,7 @@ "formatted_logging"], # These are removed from the "all" group "deprecated": ["chameleon", "django", "iterative_count", - "json_dump", "mako", "threaded_count"], + "json_dump", "mako", "threaded_count", "spitfire"], } # Calculate set of 2-and-3 compatible benchmarks. -- Repository URL: https://hg.python.org/benchmarks

1 0

Benchmark Results for Python 2.7 2015-11-27
by lp_benchmark_robot＠intel.com Nov. 27, 2015

Nov. 27, 2015

Results for project Python 2.7, build date 2015-11-27 04:49:19 +0000 commit: 0eea57ddb75f1d0a6ef3fbc762d62d016cf3c010 revision date: 2015-11-26 21:48:30 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v2.7.10, with hash 15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-) django_v2 0.41% 0.12% 2.10% 9.53% :-) pybench 0.22% 0.05% 6.16% 7.71% :-( regex_v8 0.91% 0.34% -2.49% 8.56% :-) nbody 0.13% -0.01% 9.35% 4.40% :-) json_dump_v2 0.30% -0.04% 2.12% 15.41% :-| normal_startup 1.79% 0.11% -1.50% 2.54% :-| ssbench 0.34% 0.10% 0.19% 1.90% ---------------------------------------------------------------------------------- Note: Benchmark results for ssbench are measured in requests/second while all other are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

Benchmark Results for Python Default 2015-11-27
by lp_benchmark_robot＠intel.com Nov. 27, 2015

Nov. 27, 2015

Results for project Python default, build date 2015-11-27 04:02:13 +0000 commit: 9fcfdb53e8af859b86bd8dfd84368c015f88f659 revision date: 2015-11-26 21:52:34 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v3.4.3, with hash b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-) django_v2 0.40% 1.31% 8.78% 17.17% :-| pybench 0.09% 0.02% -1.96% 9.07% :-( regex_v8 2.74% 0.19% -4.58% 5.60% :-| nbody 0.12% 0.01% 0.11% 8.87% :-| json_dump_v2 0.26% -0.49% 0.10% 11.57% :-| normal_startup 0.95% -0.22% 0.38% 4.99% ---------------------------------------------------------------------------------- Note: Benchmark results are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0