[Python-checkins] r88487 - in python/branches/release32-maint: Lib/test/test_mmap.py Misc/NEWS Modules/mmapmodule.c
antoine.pitrou
python-checkins at python.org
Tue Feb 22 00:46:27 CET 2011
Author: antoine.pitrou
Date: Tue Feb 22 00:46:27 2011
New Revision: 88487
Log:
Merged revisions 88486 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r88486 | antoine.pitrou | 2011-02-22 00:41:12 +0100 (mar., 22 févr. 2011) | 5 lines
Issue #4681: Allow mmap() to work on file sizes and offsets larger than
4GB, even on 32-bit builds. Initial patch by Ross Lagerwall, adapted for
32-bit Windows.
........
Modified:
python/branches/release32-maint/ (props changed)
python/branches/release32-maint/Lib/test/test_mmap.py
python/branches/release32-maint/Misc/NEWS
python/branches/release32-maint/Modules/mmapmodule.c
Modified: python/branches/release32-maint/Lib/test/test_mmap.py
==============================================================================
--- python/branches/release32-maint/Lib/test/test_mmap.py (original)
+++ python/branches/release32-maint/Lib/test/test_mmap.py Tue Feb 22 00:46:27 2011
@@ -1,4 +1,4 @@
-from test.support import TESTFN, run_unittest, import_module
+from test.support import TESTFN, run_unittest, import_module, unlink, requires
import unittest
import os
import re
@@ -646,9 +646,56 @@
"wrong exception raised in context manager")
self.assertTrue(m.closed, "context manager failed")
+class LargeMmapTests(unittest.TestCase):
+
+ def setUp(self):
+ unlink(TESTFN)
+
+ def tearDown(self):
+ unlink(TESTFN)
+
+ def _working_largefile(self):
+ # Only run if the current filesystem supports large files.
+ f = open(TESTFN, 'wb', buffering=0)
+ try:
+ f.seek(0x80000001)
+ f.write(b'x')
+ f.flush()
+ except (IOError, OverflowError):
+ raise unittest.SkipTest("filesystem does not have largefile support")
+ finally:
+ f.close()
+ unlink(TESTFN)
+
+ def test_large_offset(self):
+ if sys.platform[:3] == 'win' or sys.platform == 'darwin':
+ requires('largefile',
+ 'test requires %s bytes and a long time to run' % str(0x180000000))
+ self._working_largefile()
+ with open(TESTFN, 'wb') as f:
+ f.seek(0x14FFFFFFF)
+ f.write(b" ")
+
+ with open(TESTFN, 'rb') as f:
+ with mmap.mmap(f.fileno(), 0, offset=0x140000000, access=mmap.ACCESS_READ) as m:
+ self.assertEqual(m[0xFFFFFFF], 32)
+
+ def test_large_filesize(self):
+ if sys.platform[:3] == 'win' or sys.platform == 'darwin':
+ requires('largefile',
+ 'test requires %s bytes and a long time to run' % str(0x180000000))
+ self._working_largefile()
+ with open(TESTFN, 'wb') as f:
+ f.seek(0x17FFFFFFF)
+ f.write(b" ")
+
+ with open(TESTFN, 'rb') as f:
+ with mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) as m:
+ self.assertEqual(m.size(), 0x180000000)
+
def test_main():
- run_unittest(MmapTests)
+ run_unittest(MmapTests, LargeMmapTests)
if __name__ == '__main__':
test_main()
Modified: python/branches/release32-maint/Misc/NEWS
==============================================================================
--- python/branches/release32-maint/Misc/NEWS (original)
+++ python/branches/release32-maint/Misc/NEWS Tue Feb 22 00:46:27 2011
@@ -15,6 +15,10 @@
Library
-------
+- Issue #4681: Allow mmap() to work on file sizes and offsets larger than
+ 4GB, even on 32-bit builds. Initial patch by Ross Lagerwall, adapted for
+ 32-bit Windows.
+
- Issue #11089: Fix performance issue limiting the use of ConfigParser()
with large config files.
Modified: python/branches/release32-maint/Modules/mmapmodule.c
==============================================================================
--- python/branches/release32-maint/Modules/mmapmodule.c (original)
+++ python/branches/release32-maint/Modules/mmapmodule.c Tue Feb 22 00:46:27 2011
@@ -90,7 +90,11 @@
char * data;
size_t size;
size_t pos; /* relative to offset */
- size_t offset;
+#ifdef MS_WINDOWS
+ PY_LONG_LONG offset;
+#else
+ off_t offset;
+#endif
int exports;
#ifdef MS_WINDOWS
@@ -433,7 +437,11 @@
PyErr_SetFromErrno(mmap_module_error);
return NULL;
}
- return PyLong_FromSsize_t(buf.st_size);
+#ifdef HAVE_LARGEFILE_SUPPORT
+ return PyLong_FromLongLong(buf.st_size);
+#else
+ return PyLong_FromLong(buf.st_size);
+#endif
}
#endif /* UNIX */
}
@@ -467,17 +475,10 @@
CloseHandle(self->map_handle);
self->map_handle = NULL;
/* Move to the desired EOF position */
-#if SIZEOF_SIZE_T > 4
newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
off_hi = (DWORD)(self->offset >> 32);
off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
-#else
- newSizeHigh = 0;
- newSizeLow = (DWORD)(self->offset + new_size);
- off_hi = 0;
- off_lo = (DWORD)self->offset;
-#endif
SetFilePointer(self->file_handle,
newSizeLow, &newSizeHigh, FILE_BEGIN);
/* Change the size of the file */
@@ -1051,6 +1052,12 @@
}
#ifdef UNIX
+#ifdef HAVE_LARGEFILE_SUPPORT
+#define _Py_PARSE_OFF_T "L"
+#else
+#define _Py_PARSE_OFF_T "l"
+#endif
+
static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{
@@ -1058,8 +1065,9 @@
struct stat st;
#endif
mmap_object *m_obj;
- PyObject *map_size_obj = NULL, *offset_obj = NULL;
- Py_ssize_t map_size, offset;
+ PyObject *map_size_obj = NULL;
+ Py_ssize_t map_size;
+ off_t offset = 0;
int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
int devzero = -1;
int access = (int)ACCESS_DEFAULT;
@@ -1067,16 +1075,18 @@
"flags", "prot",
"access", "offset", NULL};
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii" _Py_PARSE_OFF_T, keywords,
&fd, &map_size_obj, &flags, &prot,
- &access, &offset_obj))
+ &access, &offset))
return NULL;
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
- offset = _GetMapSize(offset_obj, "offset");
- if (offset < 0)
+ if (offset < 0) {
+ PyErr_SetString(PyExc_OverflowError,
+ "memory mapped offset must be positive");
return NULL;
+ }
if ((access != (int)ACCESS_DEFAULT) &&
((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
@@ -1121,8 +1131,14 @@
"mmap offset is greater than file size");
return NULL;
}
- map_size = st.st_size - offset;
- } else if ((size_t)offset + (size_t)map_size > st.st_size) {
+ off_t calc_size = st.st_size - offset;
+ map_size = calc_size;
+ if (map_size != calc_size) {
+ PyErr_SetString(PyExc_ValueError,
+ "mmap length is too large");
+ return NULL;
+ }
+ } else if (offset + (size_t)map_size > st.st_size) {
PyErr_SetString(PyExc_ValueError,
"mmap length is greater than file size");
return NULL;
@@ -1183,12 +1199,19 @@
#endif /* UNIX */
#ifdef MS_WINDOWS
+
+/* A note on sizes and offsets: while the actual map size must hold in a
+ Py_ssize_t, both the total file size and the start offset can be longer
+ than a Py_ssize_t, so we use PY_LONG_LONG which is always 64-bit.
+*/
+
static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{
mmap_object *m_obj;
- PyObject *map_size_obj = NULL, *offset_obj = NULL;
- Py_ssize_t map_size, offset;
+ PyObject *map_size_obj = NULL;
+ Py_ssize_t map_size;
+ PY_LONG_LONG offset = 0, size;
DWORD off_hi; /* upper 32 bits of offset */
DWORD off_lo; /* lower 32 bits of offset */
DWORD size_hi; /* upper 32 bits of size */
@@ -1203,9 +1226,9 @@
"tagname",
"access", "offset", NULL };
- if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
+ if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziL", keywords,
&fileno, &map_size_obj,
- &tagname, &access, &offset_obj)) {
+ &tagname, &access, &offset)) {
return NULL;
}
@@ -1230,9 +1253,11 @@
map_size = _GetMapSize(map_size_obj, "size");
if (map_size < 0)
return NULL;
- offset = _GetMapSize(offset_obj, "offset");
- if (offset < 0)
+ if (offset < 0) {
+ PyErr_SetString(PyExc_OverflowError,
+ "memory mapped offset must be positive");
return NULL;
+ }
/* assume -1 and 0 both mean invalid filedescriptor
to 'anonymously' map memory.
@@ -1296,28 +1321,26 @@
return PyErr_SetFromWindowsErr(dwErr);
}
-#if SIZEOF_SIZE_T > 4
- m_obj->size = (((size_t)high)<<32) + low;
-#else
- if (high)
- /* File is too large to map completely */
- m_obj->size = (size_t)-1;
- else
- m_obj->size = low;
-#endif
- if (offset >= m_obj->size) {
+ size = (((PY_LONG_LONG) high) << 32) + low;
+ if (offset >= size) {
PyErr_SetString(PyExc_ValueError,
"mmap offset is greater than file size");
Py_DECREF(m_obj);
return NULL;
}
- m_obj->size -= offset;
+ if (offset - size > PY_SSIZE_T_MAX)
+ /* Map area too large to fit in memory */
+ m_obj->size = (Py_ssize_t) -1;
+ else
+ m_obj->size = (Py_ssize_t) (size - offset);
} else {
m_obj->size = map_size;
+ size = offset + map_size;
}
}
else {
m_obj->size = map_size;
+ size = offset + map_size;
}
/* set the initial position */
@@ -1338,22 +1361,10 @@
m_obj->tagname = NULL;
m_obj->access = (access_mode)access;
- /* DWORD is a 4-byte int. If we're on a box where size_t consumes
- * more than 4 bytes, we need to break it apart. Else (size_t
- * consumes 4 bytes), C doesn't define what happens if we shift
- * right by 32, so we need different code.
- */
-#if SIZEOF_SIZE_T > 4
- size_hi = (DWORD)((offset + m_obj->size) >> 32);
- size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
+ size_hi = (DWORD)(size >> 32);
+ size_lo = (DWORD)(size & 0xFFFFFFFF);
off_hi = (DWORD)(offset >> 32);
off_lo = (DWORD)(offset & 0xFFFFFFFF);
-#else
- size_hi = 0;
- size_lo = (DWORD)(offset + m_obj->size);
- off_hi = 0;
- off_lo = (DWORD)offset;
-#endif
/* For files, it would be sufficient to pass 0 as size.
For anonymous maps, we have to pass the size explicitly. */
m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
More information about the Python-checkins
mailing list