[Python-checkins] cpython (merge 3.4 -> default): Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes

[antoine.pitrou]

https://hg.python.org/cpython/rev/f0b334ae95c9
changeset:   93361:f0b334ae95c9
parent:      93358:b2a57fe50712
parent:      93360:1590c594550e
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Sun Nov 02 18:41:56 2014 +0100
summary:
  Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit platform.

files:
  Lib/test/test_bytes.py    |  13 +++++++++++++
  Misc/NEWS                 |   3 +++
  Objects/bytearrayobject.c |  23 +++++++++++++++--------
  Objects/obmalloc.c        |   8 ++++----
  4 files changed, 35 insertions(+), 12 deletions(-)


diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -13,9 +13,11 @@
 import pickle
 import tempfile
 import unittest
+
 import test.support
 import test.string_tests
 import test.buffer_tests
+from test.support import bigaddrspacetest, MAX_Py_ssize_t
 
 
 if sys.flags.bytes_warning:
@@ -111,6 +113,17 @@
         self.assertRaises(ValueError, self.type2test, [sys.maxsize+1])
         self.assertRaises(ValueError, self.type2test, [10**100])
 
+    @bigaddrspacetest
+    def test_constructor_overflow(self):
+        size = MAX_Py_ssize_t
+        self.assertRaises((OverflowError, MemoryError), self.type2test, size)
+        try:
+            # Should either pass or raise an error (e.g. on debug builds with
+            # additional malloc() overhead), but shouldn't crash.
+            bytearray(size - 4)
+        except (OverflowError, MemoryError):
+            pass
+
     def test_compare(self):
         b1 = self.type2test([1, 2, 3])
         b2 = self.type2test([1, 2, 3])
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff
+  bytes on a 32-bit platform.
+
 - Issue #22653: Fix an assertion failure in debug mode when doing a reentrant
   dict insertion in debug mode.
 
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -180,20 +180,22 @@
 }
 
 int
-PyByteArray_Resize(PyObject *self, Py_ssize_t size)
+PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size)
 {
     void *sval;
     PyByteArrayObject *obj = ((PyByteArrayObject *)self);
-    Py_ssize_t alloc = obj->ob_alloc;
-    Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes;
+    /* All computations are done unsigned to avoid integer overflows
+       (see issue #22335). */
+    size_t alloc = (size_t) obj->ob_alloc;
+    size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes);
+    size_t size = (size_t) requested_size;
 
     assert(self != NULL);
     assert(PyByteArray_Check(self));
-    assert(size >= 0);
-    assert(logical_offset >= 0);
     assert(logical_offset <= alloc);
-
-    if (size == Py_SIZE(self)) {
+    assert(requested_size >= 0);
+
+    if (requested_size == Py_SIZE(self)) {
         return 0;
     }
     if (!_canresize(obj)) {
@@ -225,6 +227,10 @@
             alloc = size + 1;
         }
     }
+    if (alloc > PY_SSIZE_T_MAX) {
+        PyErr_NoMemory();
+        return -1;
+    }
 
     if (logical_offset > 0) {
         sval = PyObject_Malloc(alloc);
@@ -232,7 +238,8 @@
             PyErr_NoMemory();
             return -1;
         }
-        memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self)));
+        memcpy(sval, PyByteArray_AS_STRING(self),
+               Py_MIN(requested_size, Py_SIZE(self)));
         PyObject_Free(obj->ob_bytes);
     }
     else {
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1828,8 +1828,8 @@
 
     bumpserialno();
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     if (use_calloc)
@@ -1909,8 +1909,8 @@
     bumpserialno();
     original_nbytes = read_size_t(q - 2*SST);
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     /* Resize and add decorations. We may get a new pointer here, in which

-- 
Repository URL: https://hg.python.org/cpython