[pypy-commit] pypy merge-2.7.2: Add large file support to mmap: 64bit offsets on 32bit platform.

amauryfa noreply at buildbot.pypy.org
Sun Jan 29 22:56:34 CET 2012


Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: merge-2.7.2
Changeset: r51958:7a82f3fbab26
Date: 2012-01-29 21:55 +0000
http://bitbucket.org/pypy/pypy/changeset/7a82f3fbab26/

Log:	Add large file support to mmap: 64bit offsets on 32bit platform.

	problem: test_large_offset() fails when interpreted, and passes when
	run on a translated pypy. Maybe a ll2ctypes issue?

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,9 +2,14 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
-from pypy.rlib import rmmap
+from pypy.rlib import rmmap, rarithmetic
 from pypy.rlib.rmmap import RValueError, RTypeError
 
+if rmmap.HAVE_LARGEFILE_SUPPORT:
+    OFF_T = rarithmetic.r_longlong
+else:
+    OFF_T = int
+
 
 class W_MMap(Wrappable):
     def __init__(self, space, mmap_obj):
@@ -55,7 +60,7 @@
             end = space.getindex_w(w_end, None)
         return space.wrap(self.mmap.find(tofind, start, end, True))
 
-    @unwrap_spec(pos='index', whence=int)
+    @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
         try:
             self.mmap.seek(pos, whence)
@@ -196,8 +201,8 @@
 
 if rmmap._POSIX:
 
-    @unwrap_spec(fileno=int, length='index', flags=int,
-                 prot=int, access=int, offset='index')
+    @unwrap_spec(fileno=int, length=int, flags=int,
+                 prot=int, access=int, offset=OFF_T)
     def mmap(space, w_subtype, fileno, length, flags=rmmap.MAP_SHARED,
              prot=rmmap.PROT_WRITE | rmmap.PROT_READ,
              access=rmmap._ACCESS_DEFAULT, offset=0):
@@ -216,8 +221,8 @@
 
 elif rmmap._MS_WINDOWS:
 
-    @unwrap_spec(fileno=int, length='index', tagname=str,
-                 access=int, offset='index')
+    @unwrap_spec(fileno=int, length=int, tagname=str,
+                 access=int, offset=OFF_T)
     def mmap(space, w_subtype, fileno, length, tagname="",
              access=rmmap._ACCESS_DEFAULT, offset=0):
         self = space.allocate_instance(W_MMap, w_subtype)
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -591,6 +591,45 @@
             assert m.size() == halfsize + 512
         m.close()
 
+    def test_large_offset(self):
+        import mmap
+        import sys
+        size = 0x14FFFFFFF
+        if sys.platform.startswith('win') or sys.platform == 'darwin':
+            self.skip('test requires %s bytes and a long time to run' % size)
+
+        with open(self.tmpname, "w+b") as f:
+            f.seek(size)
+            f.write("A")
+            f.flush()
+        with open(self.tmpname, 'rb') as f2:
+            f2.seek(size)
+            c = f2.read(1)
+            assert c == b'A'
+            m = mmap.mmap(f2.fileno(), 0, offset=0x140000000,
+                          access=mmap.ACCESS_READ)
+            try:
+                assert m[0xFFFFFFF] == b'A'
+            finally:
+                m.close()
+
+    def test_large_filesize(self):
+        import mmap
+        import sys
+        size = 0x17FFFFFFF
+        if sys.platform.startswith('win') or sys.platform == 'darwin':
+            self.skip('test requires %s bytes and a long time to run' % size)
+
+        with open(self.tmpname, "w+b") as f:
+            f.seek(size)
+            f.write(" ")
+            f.flush()
+            m = mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ)
+            try:
+                assert m.size() ==  0x180000000
+            finally:
+                m.close()
+
     def test_all(self):
         # this is a global test, ported from test_mmap.py
         import mmap
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -92,6 +92,11 @@
 
 _ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4)
 
+if rffi.sizeof(off_t) > rffi.sizeof(lltype.Signed):
+    HAVE_LARGEFILE_SUPPORT = True
+else:
+    HAVE_LARGEFILE_SUPPORT = False
+
 def external(name, args, result):
     unsafe = rffi.llexternal(name, args, result,
                              compilation_info=CConfig._compilation_info_)
@@ -428,10 +433,6 @@
         elif _POSIX:
             st = os.fstat(self.fd)
             size = st[stat.ST_SIZE]
-            if size > sys.maxint:
-                size = sys.maxint
-            else:
-                size = int(size)
         return size
 
     def write(self, data):
@@ -643,17 +644,15 @@
         else:
             mode = st[stat.ST_MODE]
             size = st[stat.ST_SIZE]
-            size -= offset
-            if size > sys.maxint:
-                size = sys.maxint
-            else:
-                size = int(size)
             if stat.S_ISREG(mode):
                 if map_size == 0:
-                    if offset > st[stat.ST_SIZE]:
-                        raise RValueError("mmap offset is greater than file size")
-                    map_size = size
-                elif map_size > size:
+                    if offset > size:
+                        raise RValueError(
+                            "mmap offset is greater than file size")
+                    map_size = int(size - offset)
+                    if map_size != size - offset:
+                        raise RValueError("mmap length is too large")
+                elif offset + map_size > size:
                     raise RValueError("mmap length is greater than file size")
 
         m = MMap(access, offset)
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -202,7 +202,7 @@
                 return cls._ptrtype
             # ctypes can raise OverflowError on 64-bit builds
             for n in [sys.maxint, 2**31]:
-                cls.MAX_SIZE = n/64
+                cls.MAX_SIZE = n / ctypes.sizeof(ctypes_item)
                 try:
                     cls._ptrtype = ctypes.POINTER(cls.MAX_SIZE * ctypes_item)
                 except OverflowError, e:


More information about the pypy-commit mailing list