[Python-checkins] [3.6] bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) (GH-8584)
Serhiy Storchaka
webhook-mailer at python.org
Tue Jul 31 05:59:04 EDT 2018
https://github.com/python/cpython/commit/ada5d99306dc8af21c32cefb3d86891e8553dbc6
commit: ada5d99306dc8af21c32cefb3d86891e8553dbc6
branch: 3.6
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2018-07-31T12:58:58+03:00
summary:
[3.6] bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) (GH-8584)
* Fix integer overflow in os.readv(), os.writev() and in os.sendfile()
with headers or trailers arguments (on BSD-based OSes and MacOS).
* Fix sending the part of the file in os.sendfile() on MacOS.
Using the trailers argument could cause sending more bytes from
the input file than was specified.
Thanks Ned Deily for testing on 32-bit MacOS.
(cherry picked from commit 9d5727326af53ddd91016d98e16ae7cf829caa95)
files:
A Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst
A Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst
M Lib/test/test_os.py
M Lib/test/test_posix.py
M Modules/posixmodule.c
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 240b7c432ba9..8339f849e84e 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -2540,12 +2540,14 @@ class Handler(asynchat.async_chat):
def __init__(self, conn):
asynchat.async_chat.__init__(self, conn)
self.in_buffer = []
+ self.accumulate = True
self.closed = False
self.push(b"220 ready\r\n")
def handle_read(self):
data = self.recv(4096)
- self.in_buffer.append(data)
+ if self.accumulate:
+ self.in_buffer.append(data)
def get_data(self):
return b''.join(self.in_buffer)
@@ -2627,6 +2629,8 @@ class TestSendfile(unittest.TestCase):
not sys.platform.startswith("sunos")
requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS,
'requires headers and trailers support')
+ requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
+ 'test is only meaningful on 32-bit builds')
@classmethod
def setUpClass(cls):
@@ -2657,17 +2661,13 @@ def tearDown(self):
self.server.stop()
self.server = None
- def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
+ def sendfile_wrapper(self, *args, **kwargs):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
- while 1:
+ while True:
try:
- if self.SUPPORT_HEADERS_TRAILERS:
- return os.sendfile(sock, file, offset, nbytes, headers,
- trailers)
- else:
- return os.sendfile(sock, file, offset, nbytes)
+ return os.sendfile(*args, **kwargs)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
@@ -2758,20 +2758,22 @@ def test_keywords(self):
@requires_headers_trailers
def test_headers(self):
total_sent = 0
+ expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1]
sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
- headers=[b"x" * 512])
+ headers=[b"x" * 512, b"y" * 256])
+ self.assertLessEqual(sent, 512 + 256 + 4096)
total_sent += sent
offset = 4096
- nbytes = 4096
- while 1:
+ while total_sent < len(expected_data):
+ nbytes = min(len(expected_data) - total_sent, 4096)
sent = self.sendfile_wrapper(self.sockno, self.fileno,
offset, nbytes)
if sent == 0:
break
+ self.assertLessEqual(sent, nbytes)
total_sent += sent
offset += sent
- expected_data = b"x" * 512 + self.DATA
self.assertEqual(total_sent, len(expected_data))
self.client.close()
self.server.wait()
@@ -2787,12 +2789,30 @@ def test_trailers(self):
create_file(TESTFN2, file_data)
with open(TESTFN2, 'rb') as f:
- os.sendfile(self.sockno, f.fileno(), 0, len(file_data),
- trailers=[b"1234"])
+ os.sendfile(self.sockno, f.fileno(), 0, 5,
+ trailers=[b"123456", b"789"])
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
- self.assertEqual(data, b"abcdef1234")
+ self.assertEqual(data, b"abcde123456789")
+
+ @requires_headers_trailers
+ @requires_32b
+ def test_headers_overflow_32bits(self):
+ self.server.handler_instance.accumulate = False
+ with self.assertRaises(OSError) as cm:
+ os.sendfile(self.sockno, self.fileno, 0, 0,
+ headers=[b"x" * 2**16] * 2**15)
+ self.assertEqual(cm.exception.errno, errno.EINVAL)
+
+ @requires_headers_trailers
+ @requires_32b
+ def test_trailers_overflow_32bits(self):
+ self.server.handler_instance.accumulate = False
+ with self.assertRaises(OSError) as cm:
+ os.sendfile(self.sockno, self.fileno, 0, 0,
+ trailers=[b"x" * 2**16] * 2**15)
+ self.assertEqual(cm.exception.errno, errno.EINVAL)
@requires_headers_trailers
@unittest.skipUnless(hasattr(os, 'SF_NODISKIO'),
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index eb3ef8494788..a204377d9af4 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -19,6 +19,9 @@
_DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(),
support.TESTFN + '-dummy-symlink')
+requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
+ 'test is only meaningful on 32-bit builds')
+
class PosixTester(unittest.TestCase):
def setUp(self):
@@ -322,6 +325,17 @@ def test_writev(self):
finally:
os.close(fd)
+ @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()")
+ @requires_32b
+ def test_writev_overflow_32bits(self):
+ fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+ try:
+ with self.assertRaises(OSError) as cm:
+ os.writev(fd, [b"x" * 2**16] * 2**15)
+ self.assertEqual(cm.exception.errno, errno.EINVAL)
+ finally:
+ os.close(fd)
+
@unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()")
def test_readv(self):
fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
@@ -344,6 +358,19 @@ def test_readv(self):
finally:
os.close(fd)
+ @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()")
+ @requires_32b
+ def test_readv_overflow_32bits(self):
+ fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT)
+ try:
+ buf = [bytearray(2**16)] * 2**15
+ with self.assertRaises(OSError) as cm:
+ os.readv(fd, buf)
+ self.assertEqual(cm.exception.errno, errno.EINVAL)
+ self.assertEqual(bytes(buf[0]), b'\0'* 2**16)
+ finally:
+ os.close(fd)
+
@unittest.skipUnless(hasattr(posix, 'dup'),
'test needs posix.dup()')
def test_dup(self):
diff --git a/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst
new file mode 100644
index 000000000000..26210fa9b92c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst
@@ -0,0 +1,3 @@
+Fixed integer overflow in :func:`os.readv` and :func:`os.writev`
+and in :func:`os.sendfile` with *headers* or *trailers* arguments
+(on BSD-based OSes and macOS).
diff --git a/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst
new file mode 100644
index 000000000000..547342c2e9dd
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst
@@ -0,0 +1,3 @@
+Fixed sending the part of the file in :func:`os.sendfile` on macOS. Using
+the *trailers* argument could cause sending more bytes from the input file
+than was specified.
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 9c05acb24f0c..af6b9dc4ef03 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7924,11 +7924,10 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length)
#if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \
|| defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV)
-static Py_ssize_t
+static int
iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type)
{
Py_ssize_t i, j;
- Py_ssize_t blen, total = 0;
*iov = PyMem_New(struct iovec, cnt);
if (*iov == NULL) {
@@ -7953,11 +7952,9 @@ iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, in
}
Py_DECREF(item);
(*iov)[i].iov_base = (*buf)[i].buf;
- blen = (*buf)[i].len;
- (*iov)[i].iov_len = blen;
- total += blen;
+ (*iov)[i].iov_len = (*buf)[i].len;
}
- return total;
+ return 0;
fail:
PyMem_Del(*iov);
@@ -8166,12 +8163,20 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict)
}
if (i > 0) {
sf.hdr_cnt = (int)i;
- i = iov_setup(&(sf.headers), &hbuf,
- headers, sf.hdr_cnt, PyBUF_SIMPLE);
- if (i < 0)
+ if (iov_setup(&(sf.headers), &hbuf,
+ headers, sf.hdr_cnt, PyBUF_SIMPLE) < 0)
return NULL;
#ifdef __APPLE__
- sbytes += i;
+ for (i = 0; i < sf.hdr_cnt; i++) {
+ Py_ssize_t blen = sf.headers[i].iov_len;
+# define OFF_T_MAX 0x7fffffffffffffff
+ if (sbytes >= OFF_T_MAX - blen) {
+ PyErr_SetString(PyExc_OverflowError,
+ "sendfile() header is too large");
+ return NULL;
+ }
+ sbytes += blen;
+ }
#endif
}
}
@@ -8192,13 +8197,9 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict)
}
if (i > 0) {
sf.trl_cnt = (int)i;
- i = iov_setup(&(sf.trailers), &tbuf,
- trailers, sf.trl_cnt, PyBUF_SIMPLE);
- if (i < 0)
+ if (iov_setup(&(sf.trailers), &tbuf,
+ trailers, sf.trl_cnt, PyBUF_SIMPLE) < 0)
return NULL;
-#ifdef __APPLE__
- sbytes += i;
-#endif
}
}
}
More information about the Python-checkins
mailing list