Mailman 3 November 2014 - Python-checkins

cpython (2.7): Issue #22775: Fixed unpickling of Cookie.SimpleCookie with protocol 2.
by serhiy.storchaka 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/515331e0ca0c changeset: 93368:515331e0ca0c branch: 2.7 parent: 93349:16dfefe67c1f user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 02 22:35:47 2014 +0200 summary: Issue #22775: Fixed unpickling of Cookie.SimpleCookie with protocol 2. Patch by Tim Graham. files: Lib/Cookie.py | 8 ++++++-- Lib/test/test_cookie.py | 13 +++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/Cookie.py b/Lib/Cookie.py --- a/Lib/Cookie.py +++ b/Lib/Cookie.py @@ -591,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): diff --git a/Lib/test/test_cookie.py b/Lib/test/test_cookie.py --- a/Lib/test/test_cookie.py +++ b/Lib/test/test_cookie.py @@ -3,6 +3,7 @@ from test.test_support import run_unittest, run_doctest, check_warnings import unittest import Cookie +import pickle class CookieTests(unittest.TestCase): @@ -141,6 +142,18 @@ self.assertEqual(dict(C), {}) self.assertEqual(C.output(), '') + def test_pickle(self): + rawdata = 'Customer="WILE_E_COYOTE"; Path=/acme; Version=1' + expected_output = 'Set-Cookie: %s' % rawdata + + C = Cookie.SimpleCookie() + C.load(rawdata) + self.assertEqual(C.output(), expected_output) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + C1 = pickle.loads(pickle.dumps(C, protocol=proto)) + self.assertEqual(C1.output(), expected_output) + def test_main(): run_unittest(CookieTests) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -484,6 +484,7 @@ Shelley Gooch David Goodger Hans de Graaff +Tim Graham Nathaniel Gray Eddy De Greef Grant Griffin diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -37,6 +37,9 @@ Library ------- +- Issue #22775: Fixed unpickling of Cookie.SimpleCookie with protocol 2. + Patch by Tim Graham. + - Issue #22776: Brought excluded code into the scope of a try block in SysLogHandler.emit(). -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2
by serhiy.storchaka 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/caa8f9248ab8 changeset: 93367:caa8f9248ab8 parent: 93365:21b4356ffc06 parent: 93366:7be6ef737aaf user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 02 22:19:56 2014 +0200 summary: Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 and above. Patch by Tim Graham. files: Lib/http/cookies.py | 8 ++++++-- Lib/test/pickletester.py | 2 +- Lib/test/test_http_cookies.py | 15 ++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -486,8 +486,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): """Return a string suitable for HTTP.""" diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1284,7 +1284,7 @@ loaded = self.loads(DATA5) self.assertEqual(type(loaded), SimpleCookie) self.assertEqual(list(loaded.keys()), ["key"]) - self.assertEqual(loaded["key"].value, "Set-Cookie: key=value") + self.assertEqual(loaded["key"].value, "value") for (exc, data) in DATA7.items(): loaded = self.loads(data) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -3,7 +3,7 @@ from test.support import run_unittest, run_doctest, check_warnings import unittest from http import cookies - +import pickle import warnings class CookieTests(unittest.TestCase): @@ -187,6 +187,19 @@ self.assertEqual(dict(C), {}) self.assertEqual(C.output(), '') + def test_pickle(self): + rawdata = 'Customer="WILE_E_COYOTE"; Path=/acme; Version=1' + expected_output = 'Set-Cookie: %s' % rawdata + + C = cookies.SimpleCookie() + C.load(rawdata) + self.assertEqual(C.output(), expected_output) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + C1 = pickle.loads(pickle.dumps(C, protocol=proto)) + self.assertEqual(C1.output(), expected_output) + class MorselTests(unittest.TestCase): """Tests for the Morsel object.""" diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -496,6 +496,7 @@ Shelley Gooch David Goodger Hans de Graaff +Tim Graham Kim Gräsman Nathaniel Gray Eddy De Greef diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -183,6 +183,9 @@ Library ------- +- Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 + and above. Patch by Tim Graham. + - Issue #22776: Brought excluded code into the scope of a try block in SysLogHandler.emit(). -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2
by serhiy.storchaka 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/7be6ef737aaf changeset: 93366:7be6ef737aaf branch: 3.4 parent: 93364:c51d85cf57f2 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 02 22:18:25 2014 +0200 summary: Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 and above. Patch by Tim Graham. files: Lib/http/cookies.py | 8 ++++++-- Lib/test/pickletester.py | 2 +- Lib/test/test_http_cookies.py | 15 ++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -486,8 +486,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): """Return a string suitable for HTTP.""" diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1284,7 +1284,7 @@ loaded = self.loads(DATA5) self.assertEqual(type(loaded), SimpleCookie) self.assertEqual(list(loaded.keys()), ["key"]) - self.assertEqual(loaded["key"].value, "Set-Cookie: key=value") + self.assertEqual(loaded["key"].value, "value") for (exc, data) in DATA7.items(): loaded = self.loads(data) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -3,7 +3,7 @@ from test.support import run_unittest, run_doctest, check_warnings import unittest from http import cookies - +import pickle import warnings class CookieTests(unittest.TestCase): @@ -187,6 +187,19 @@ self.assertEqual(dict(C), {}) self.assertEqual(C.output(), '') + def test_pickle(self): + rawdata = 'Customer="WILE_E_COYOTE"; Path=/acme; Version=1' + expected_output = 'Set-Cookie: %s' % rawdata + + C = cookies.SimpleCookie() + C.load(rawdata) + self.assertEqual(C.output(), expected_output) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + C1 = pickle.loads(pickle.dumps(C, protocol=proto)) + self.assertEqual(C1.output(), expected_output) + class MorselTests(unittest.TestCase): """Tests for the Morsel object.""" diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -492,6 +492,7 @@ Shelley Gooch David Goodger Hans de Graaff +Tim Graham Nathaniel Gray Eddy De Greef Grant Griffin diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Library ------- +- Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 + and above. Patch by Tim Graham. + - Issue #22366: urllib.request.urlopen will accept a context object (SSLContext) as an argument which will then used be for HTTPS connection. Patch by Alex Gaynor. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): merge 3.4
by benjamin.peterson 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/21b4356ffc06 changeset: 93365:21b4356ffc06 parent: 93362:7e89c24245ea parent: 93364:c51d85cf57f2 user: Benjamin Peterson <benjamin(a)python.org> date: Sun Nov 02 13:19:15 2014 -0500 summary: merge 3.4 files: Doc/library/urllib.request.rst | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -48,8 +48,8 @@ only works for HTTP, HTTPS and FTP connections. If *context* is specified, it must be a :class:`ssl.SSLContext` instance - describing the various SSL options. See - :class:`~http.client.HTTPSConnection` for more details. + describing the various SSL options. See :class:`~http.client.HTTPSConnection` + for more details. The optional *cafile* and *capath* parameters specify a set of trusted CA certificates for HTTPS requests. *cafile* should point to a single @@ -118,6 +118,7 @@ .. versionchanged:: 3.5 *context* was added. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): wrap
by benjamin.peterson 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/c51d85cf57f2 changeset: 93364:c51d85cf57f2 branch: 3.4 user: Benjamin Peterson <benjamin(a)python.org> date: Sun Nov 02 13:17:56 2014 -0500 summary: wrap files: Doc/library/urllib.request.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -48,8 +48,8 @@ only works for HTTP, HTTPS and FTP connections. If *context* is specified, it must be a :class:`ssl.SSLContext` instance - describing the various SSL options. See - :class:`~http.client.HTTPSConnection` for more details. + describing the various SSL options. See :class:`~http.client.HTTPSConnection` + for more details. The optional *cafile* and *capath* parameters specify a set of trusted CA certificates for HTTPS requests. *cafile* should point to a single -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): backport context argument of urlopen (#22366) for pep 476
by benjamin.peterson 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/13f46fc1a002 changeset: 93363:13f46fc1a002 branch: 3.4 parent: 93360:1590c594550e user: Senthil Kumaran <senthil(a)uthcode.com> date: Fri Sep 19 15:23:30 2014 +0800 summary: backport context argument of urlopen (#22366) for pep 476 files: Doc/library/urllib.request.rst | 9 ++++++++- Lib/test/test_urllib.py | 8 ++++++++ Lib/urllib/request.py | 10 +++++++++- Misc/NEWS | 4 ++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -16,7 +16,7 @@ The :mod:`urllib.request` module defines the following functions: -.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False) +.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False, context=None) Open the URL *url*, which can be either a string or a :class:`Request` object. @@ -47,6 +47,10 @@ the global default timeout setting will be used). This actually only works for HTTP, HTTPS and FTP connections. + If *context* is specified, it must be a :class:`ssl.SSLContext` instance + describing the various SSL options. See + :class:`~http.client.HTTPSConnection` for more details. + The optional *cafile* and *capath* parameters specify a set of trusted CA certificates for HTTPS requests. *cafile* should point to a single file containing a bundle of CA certificates, whereas *capath* should @@ -111,6 +115,9 @@ .. versionchanged:: 3.3 *cadefault* was added. + .. versionchanged:: 3.4.3 + *context* was added. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -10,6 +10,7 @@ from unittest.mock import patch from test import support import os +import ssl import sys import tempfile from nturl2path import url2pathname, pathname2url @@ -379,6 +380,13 @@ with support.check_warnings(('',DeprecationWarning)): urllib.request.URLopener() + def test_cafile_and_context(self): + context = ssl.create_default_context() + with self.assertRaises(ValueError): + urllib.request.urlopen( + "https://localhost", cafile="/nonexistent/path", context=context + ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -136,9 +136,14 @@ _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - *, cafile=None, capath=None, cadefault=False): + *, cafile=None, capath=None, cadefault=False, context=None): global _opener if cafile or capath or cadefault: + if context is not None: + raise ValueError( + "You can't pass both context and any of cafile, capath, and " + "cadefault" + ) if not _have_ssl: raise ValueError('SSL support not available') context = ssl._create_stdlib_context(cert_reqs=ssl.CERT_REQUIRED, @@ -146,6 +151,9 @@ capath=capath) https_handler = HTTPSHandler(context=context, check_hostname=True) opener = build_opener(https_handler) + elif context: + https_handler = HTTPSHandler(context=context) + opener = build_opener(https_handler) elif _opener is None: _opener = opener = build_opener() else: diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Library ------- +- Issue #22366: urllib.request.urlopen will accept a context object + (SSLContext) as an argument which will then used be for HTTPS connection. + Patch by Alex Gaynor. + - Issue #22776: Brought excluded code into the scope of a try block in SysLogHandler.emit(). -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge default -> default): Merge heads
by antoine.pitrou 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/7e89c24245ea changeset: 93362:7e89c24245ea parent: 93361:f0b334ae95c9 parent: 93359:016bc54999a2 user: Antoine Pitrou <solipsis(a)pitrou.net> date: Sun Nov 02 18:42:41 2014 +0100 summary: Merge heads files: Lib/test/test_urllib.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -10,7 +10,10 @@ from unittest.mock import patch from test import support import os -import ssl +try: + import ssl +except ImportError: + ssl = None import sys import tempfile from nturl2path import url2pathname, pathname2url @@ -380,6 +383,7 @@ with support.check_warnings(('',DeprecationWarning)): urllib.request.URLopener() + @unittest.skipUnless(ssl, "ssl module required") def test_cafile_and_context(self): context = ssl.create_default_context() with self.assertRaises(ValueError): -- Repository URL: https://hg.python.org/cpython

1 0

cpython: Fix test_urllib without the ssl module
by antoine.pitrou 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/016bc54999a2 changeset: 93359:016bc54999a2 parent: 93347:64a54f0c87d7 user: Antoine Pitrou <solipsis(a)pitrou.net> date: Sun Nov 02 17:23:14 2014 +0100 summary: Fix test_urllib without the ssl module files: Lib/test/test_urllib.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -10,7 +10,10 @@ from unittest.mock import patch from test import support import os -import ssl +try: + import ssl +except ImportError: + ssl = None import sys import tempfile from nturl2path import url2pathname, pathname2url @@ -380,6 +383,7 @@ with support.check_warnings(('',DeprecationWarning)): urllib.request.URLopener() + @unittest.skipUnless(ssl, "ssl module required") def test_cafile_and_context(self): context = ssl.create_default_context() with self.assertRaises(ValueError): -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes
by antoine.pitrou 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/1590c594550e changeset: 93360:1590c594550e branch: 3.4 parent: 93357:4924b0ce72c0 user: Antoine Pitrou <solipsis(a)pitrou.net> date: Sun Nov 02 18:40:09 2014 +0100 summary: Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit platform. files: Lib/test/test_bytes.py | 13 +++++++++++++ Misc/NEWS | 3 +++ Objects/bytearrayobject.c | 23 +++++++++++++++-------- Objects/obmalloc.c | 8 ++++---- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -13,9 +13,11 @@ import pickle import tempfile import unittest + import test.support import test.string_tests import test.buffer_tests +from test.support import bigaddrspacetest, MAX_Py_ssize_t if sys.flags.bytes_warning: @@ -111,6 +113,17 @@ self.assertRaises(ValueError, self.type2test, [sys.maxsize+1]) self.assertRaises(ValueError, self.type2test, [10**100]) + @bigaddrspacetest + def test_constructor_overflow(self): + size = MAX_Py_ssize_t + self.assertRaises((OverflowError, MemoryError), self.type2test, size) + try: + # Should either pass or raise an error (e.g. on debug builds with + # additional malloc() overhead), but shouldn't crash. + bytearray(size - 4) + except (OverflowError, MemoryError): + pass + def test_compare(self): b1 = self.type2test([1, 2, 3]) b2 = self.type2test([1, 2, 3]) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,6 +11,9 @@ Core and Builtins ----------------- +- Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff + bytes on a 32-bit platform. + - Issue #22653: Fix an assertion failure in debug mode when doing a reentrant dict insertion in debug mode. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -175,20 +175,22 @@ } int -PyByteArray_Resize(PyObject *self, Py_ssize_t size) +PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) { void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); - Py_ssize_t alloc = obj->ob_alloc; - Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes; + /* All computations are done unsigned to avoid integer overflows + (see issue #22335). */ + size_t alloc = (size_t) obj->ob_alloc; + size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes); + size_t size = (size_t) requested_size; assert(self != NULL); assert(PyByteArray_Check(self)); - assert(size >= 0); - assert(logical_offset >= 0); assert(logical_offset <= alloc); - - if (size == Py_SIZE(self)) { + assert(requested_size >= 0); + + if (requested_size == Py_SIZE(self)) { return 0; } if (!_canresize(obj)) { @@ -220,6 +222,10 @@ alloc = size + 1; } } + if (alloc > PY_SSIZE_T_MAX) { + PyErr_NoMemory(); + return -1; + } if (logical_offset > 0) { sval = PyObject_Malloc(alloc); @@ -227,7 +233,8 @@ PyErr_NoMemory(); return -1; } - memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self))); + memcpy(sval, PyByteArray_AS_STRING(self), + Py_MIN(requested_size, Py_SIZE(self))); PyObject_Free(obj->ob_bytes); } else { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1754,8 +1754,8 @@ bumpserialno(); total = nbytes + 4*SST; - if (total < nbytes) - /* overflow: can't represent total as a size_t */ + if (nbytes > PY_SSIZE_T_MAX - 4*SST) + /* overflow: can't represent total as a Py_ssize_t */ return NULL; p = (uchar *)api->alloc.malloc(api->alloc.ctx, total); @@ -1817,8 +1817,8 @@ bumpserialno(); original_nbytes = read_size_t(q - 2*SST); total = nbytes + 4*SST; - if (total < nbytes) - /* overflow: can't represent total as a size_t */ + if (nbytes > PY_SSIZE_T_MAX - 4*SST) + /* overflow: can't represent total as a Py_ssize_t */ return NULL; /* Resize and add decorations. We may get a new pointer here, in which -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes
by antoine.pitrou 03 Nov '14

03 Nov '14

https://hg.python.org/cpython/rev/f0b334ae95c9 changeset: 93361:f0b334ae95c9 parent: 93358:b2a57fe50712 parent: 93360:1590c594550e user: Antoine Pitrou <solipsis(a)pitrou.net> date: Sun Nov 02 18:41:56 2014 +0100 summary: Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit platform. files: Lib/test/test_bytes.py | 13 +++++++++++++ Misc/NEWS | 3 +++ Objects/bytearrayobject.c | 23 +++++++++++++++-------- Objects/obmalloc.c | 8 ++++---- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -13,9 +13,11 @@ import pickle import tempfile import unittest + import test.support import test.string_tests import test.buffer_tests +from test.support import bigaddrspacetest, MAX_Py_ssize_t if sys.flags.bytes_warning: @@ -111,6 +113,17 @@ self.assertRaises(ValueError, self.type2test, [sys.maxsize+1]) self.assertRaises(ValueError, self.type2test, [10**100]) + @bigaddrspacetest + def test_constructor_overflow(self): + size = MAX_Py_ssize_t + self.assertRaises((OverflowError, MemoryError), self.type2test, size) + try: + # Should either pass or raise an error (e.g. on debug builds with + # additional malloc() overhead), but shouldn't crash. + bytearray(size - 4) + except (OverflowError, MemoryError): + pass + def test_compare(self): b1 = self.type2test([1, 2, 3]) b2 = self.type2test([1, 2, 3]) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff + bytes on a 32-bit platform. + - Issue #22653: Fix an assertion failure in debug mode when doing a reentrant dict insertion in debug mode. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -180,20 +180,22 @@ } int -PyByteArray_Resize(PyObject *self, Py_ssize_t size) +PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) { void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); - Py_ssize_t alloc = obj->ob_alloc; - Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes; + /* All computations are done unsigned to avoid integer overflows + (see issue #22335). */ + size_t alloc = (size_t) obj->ob_alloc; + size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes); + size_t size = (size_t) requested_size; assert(self != NULL); assert(PyByteArray_Check(self)); - assert(size >= 0); - assert(logical_offset >= 0); assert(logical_offset <= alloc); - - if (size == Py_SIZE(self)) { + assert(requested_size >= 0); + + if (requested_size == Py_SIZE(self)) { return 0; } if (!_canresize(obj)) { @@ -225,6 +227,10 @@ alloc = size + 1; } } + if (alloc > PY_SSIZE_T_MAX) { + PyErr_NoMemory(); + return -1; + } if (logical_offset > 0) { sval = PyObject_Malloc(alloc); @@ -232,7 +238,8 @@ PyErr_NoMemory(); return -1; } - memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self))); + memcpy(sval, PyByteArray_AS_STRING(self), + Py_MIN(requested_size, Py_SIZE(self))); PyObject_Free(obj->ob_bytes); } else { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1828,8 +1828,8 @@ bumpserialno(); total = nbytes + 4*SST; - if (total < nbytes) - /* overflow: can't represent total as a size_t */ + if (nbytes > PY_SSIZE_T_MAX - 4*SST) + /* overflow: can't represent total as a Py_ssize_t */ return NULL; if (use_calloc) @@ -1909,8 +1909,8 @@ bumpserialno(); original_nbytes = read_size_t(q - 2*SST); total = nbytes + 4*SST; - if (total < nbytes) - /* overflow: can't represent total as a size_t */ + if (nbytes > PY_SSIZE_T_MAX - 4*SST) + /* overflow: can't represent total as a Py_ssize_t */ return NULL; /* Resize and add decorations. We may get a new pointer here, in which -- Repository URL: https://hg.python.org/cpython

1 0