[pypy-svn] r51901 - in pypy/dist/pypy: module/_file module/_file/test module/posix/test rlib rpython/module translator/c/test
arigo at codespeak.net
arigo at codespeak.net
Wed Feb 27 14:54:40 CET 2008
Author: arigo
Date: Wed Feb 27 14:54:38 2008
New Revision: 51901
Added:
pypy/dist/pypy/module/_file/test/test_large_file.py
pypy/dist/pypy/module/posix/test/__init__.py
Modified:
pypy/dist/pypy/module/_file/interp_file.py
pypy/dist/pypy/module/posix/test/test_posix2.py
pypy/dist/pypy/rlib/streamio.py
pypy/dist/pypy/rpython/module/ll_os_stat.py
pypy/dist/pypy/translator/c/test/test_extfunc.py
pypy/dist/pypy/translator/c/test/test_standalone.py
Log:
issue357 testing
Support for 64-bit offsets in 'file' objects.
Also includes a test for r51899 in test_standalone.
(Note that r51899 broke test_extfunc in an obscure way)
Modified: pypy/dist/pypy/module/_file/interp_file.py
==============================================================================
--- pypy/dist/pypy/module/_file/interp_file.py (original)
+++ pypy/dist/pypy/module/_file/interp_file.py Wed Feb 27 14:54:38 2008
@@ -1,6 +1,7 @@
import py
import os
from pypy.rlib import streamio
+from pypy.rlib.rarithmetic import r_longlong
from pypy.module._file.interp_stream import W_AbstractStream
from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror
from pypy.interpreter.error import OperationError
@@ -177,7 +178,7 @@
if w_size is None or space.is_w(w_size, space.w_None):
size = stream.tell()
else:
- size = space.int_w(w_size)
+ size = space.r_longlong_w(w_size)
stream.truncate(size)
def direct_write(self, data):
@@ -309,7 +310,7 @@
total number of bytes in the lines returned.""",
wrapresult = "wrap_list_of_str(space, result)")
- _decl(locals(), "seek", ['self', int, int],
+ _decl(locals(), "seek", ['self', r_longlong, int],
"""seek(offset[, whence]) -> None. Move to new file position.
Argument offset is a byte count. Optional argument whence defaults to
Added: pypy/dist/pypy/module/_file/test/test_large_file.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/_file/test/test_large_file.py Wed Feb 27 14:54:38 2008
@@ -0,0 +1,69 @@
+import py
+
+from pypy.conftest import gettestobjspace
+
+class AppTestLargeFile(object):
+ def setup_class(cls):
+ cls.space = gettestobjspace(usemodules=("_file", ))
+ cls.w_temppath = cls.space.wrap(
+ str(py.test.ensuretemp("fileimpl").join("large.data")))
+
+ def setup_method(self, meth):
+ if getattr(meth, 'need_sparse_files', False):
+ from pypy.module.posix.test.test_posix2 import need_sparse_files
+ need_sparse_files()
+
+ def test_large_seek_offsets(self):
+ import _file
+ FAR = 0x122223333
+ f = _file.file(self.temppath, "w+b")
+ f.write("hello world")
+ f.seek(FAR)
+ assert f.tell() == FAR
+ f.seek(-10, 1)
+ assert f.tell() == FAR - 10
+ f.seek(0x123456789, 1)
+ assert f.tell() == FAR - 10 + 0x123456789
+ f.seek(-FAR, 1)
+ assert f.tell() == -10 + 0x123456789
+ f.seek(FAR, 2)
+ assert f.tell() == len("hello world") + FAR
+ f.close()
+
+ def test_large_sparse(self):
+ import _file
+ FAR = 0x122223333
+ f = _file.file(self.temppath, "w+b")
+ f.seek(FAR)
+ f.write('end')
+ f.seek(0)
+ data = f.read(1234)
+ assert data == 1234 * '\x00'
+ f.seek(FAR-2-1234, 1)
+ data = f.read(4321)
+ assert data == '\x00\x00end'
+ f.seek(-1, 2)
+ assert f.tell() == FAR + 2
+ f.truncate()
+ f.seek(0, 2)
+ assert f.tell() == FAR + 2
+ f.truncate(FAR + 1)
+ f.seek(FAR-2, 0)
+ data = f.read(1)
+ assert data == '\x00'
+ assert f.tell() == FAR - 1
+ data = f.read(1)
+ assert data == '\x00'
+ data = f.read(1)
+ assert data == 'e'
+ data = f.read(1)
+ assert data == ''
+ assert f.tell() == FAR + 1
+ import sys
+ if FAR > sys.maxint:
+ f.seek(0)
+ raises(OverflowError, f.read, FAR)
+ raises(OverflowError, f.readline, FAR)
+ raises(OverflowError, f.readlines, FAR)
+ f.close()
+ test_large_sparse.need_sparse_files = True
Added: pypy/dist/pypy/module/posix/test/__init__.py
==============================================================================
Modified: pypy/dist/pypy/module/posix/test/test_posix2.py
==============================================================================
--- pypy/dist/pypy/module/posix/test/test_posix2.py (original)
+++ pypy/dist/pypy/module/posix/test/test_posix2.py Wed Feb 27 14:54:38 2008
@@ -19,6 +19,12 @@
pdir.join('another_longer_file_name').write("test3")
mod.pdir = pdir
+def need_sparse_files():
+ if sys.platform == 'darwin':
+ py.test.skip("no sparse files on default Mac OS X file system")
+ if os.name == 'nt':
+ py.test.skip("no sparse files on Windows")
+
class AppTestPosix:
def setup_class(cls):
cls.space = space
@@ -31,6 +37,10 @@
cls.w_geteuid = space.wrap(os.geteuid())
if hasattr(os, 'getgid'):
cls.w_getgid = space.wrap(os.getgid())
+
+ def setup_method(self, meth):
+ if getattr(meth, 'need_sparse_files', False):
+ need_sparse_files()
def test_posix_is_pypy_s(self):
assert self.posix.__file__
@@ -296,11 +306,6 @@
def test_largefile(self):
os = self.posix
- import sys
- if sys.platform == 'darwin':
- skip("no sparse files on default Mac OS X file system")
- if os.__name__ == 'nt':
- skip("no sparse files on Windows")
fd = os.open(self.path2, os.O_RDWR | os.O_CREAT, 0666)
os.ftruncate(fd, 10000000000L)
res = os.lseek(fd, 9900000000L, 0)
@@ -313,6 +318,7 @@
st = os.stat(self.path2)
assert st.st_size == 10000000000L
+ test_largefile.need_sparse_files = True
class AppTestEnvironment(object):
def setup_class(cls):
Modified: pypy/dist/pypy/rlib/streamio.py
==============================================================================
--- pypy/dist/pypy/rlib/streamio.py (original)
+++ pypy/dist/pypy/rlib/streamio.py Wed Feb 27 14:54:38 2008
@@ -29,7 +29,16 @@
"""
+#
+# File offsets are all 'r_longlong', but a single read or write cannot
+# transfer more data that fits in an RPython 'int' (because that would not
+# fit in a single string anyway). This module needs to be careful about
+# where r_longlong values end up: as argument to seek() and truncate() and
+# return value of tell(), but not as argument to read().
+#
+
import os, sys
+from pypy.rlib.rarithmetic import r_longlong, intmask
from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC
O_BINARY = getattr(os, "O_BINARY", 0)
@@ -226,10 +235,10 @@
os.lseek(self.fd, offset, whence)
def tell(self):
- #XXX we really want r_longlong
- return int(os.lseek(self.fd, 0, 1))
+ return os.lseek(self.fd, 0, 1)
def read(self, n):
+ assert isinstance(n, int)
return os.read(self.fd, n)
def write(self, data):
@@ -298,6 +307,7 @@
return data
def read(self, n):
+ assert isinstance(n, int)
end = self.pos + n
data = self.mm[self.pos:end]
if not data:
@@ -356,10 +366,10 @@
("read", [int]),
("write", [str]),
("tell", []),
- ("seek", ["index", int]),
+ ("seek", [r_longlong, int]),
("readall", []),
("readline", []),
- ("truncate", [int]),
+ ("truncate", [r_longlong]),
("flush", []),
("flushable", []),
("close", []),
@@ -389,6 +399,14 @@
return d[meth_name]
+def offset2int(offset):
+ intoffset = intmask(offset)
+ if intoffset != offset:
+ raise StreamError("seek() from a non-seekable source:"
+ " this would read and discard more"
+ " than sys.maxint bytes")
+ return intoffset
+
class BufferingInputStream(Stream):
"""Standard buffering input stream.
@@ -450,22 +468,25 @@
while self.lines:
line = self.lines[0]
if offset <= len(line):
- assert offset >= 0
- self.lines[0] = line[offset:]
+ intoffset = intmask(offset)
+ assert intoffset >= 0
+ self.lines[0] = line[intoffset:]
return
offset -= len(self.lines[0]) - 1
del self.lines[0]
assert not self.lines
if offset <= len(self.buf):
- assert offset >= 0
- self.buf = self.buf[offset:]
+ intoffset = intmask(offset)
+ assert intoffset >= 0
+ self.buf = self.buf[intoffset:]
return
offset -= len(self.buf)
self.buf = ""
try:
self.do_seek(offset, 1)
except NotImplementedError:
- self.read(offset)
+ intoffset = offset2int(offset)
+ self.read(intoffset)
return
if whence == 2:
try:
@@ -478,6 +499,7 @@
return
# Skip relative to EOF by reading and saving only just as
# much as needed
+ intoffset = offset2int(offset)
data = "\n".join(self.lines + [self.buf])
total = len(data)
buffers = [data]
@@ -489,10 +511,10 @@
break
buffers.append(data)
total += len(data)
- while buffers and total >= len(buffers[0]) - offset:
+ while buffers and total >= len(buffers[0]) - intoffset:
total -= len(buffers[0])
del buffers[0]
- cutoff = total + offset
+ cutoff = total + intoffset
if cutoff < 0:
raise StreamError("cannot seek back")
if buffers:
@@ -517,6 +539,7 @@
return "".join(more)
def read(self, n):
+ assert isinstance(n, int)
assert n >= 0
if self.lines:
# See if this can be satisfied from self.lines[0]
Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py
==============================================================================
--- pypy/dist/pypy/rpython/module/ll_os_stat.py (original)
+++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Feb 27 14:54:38 2008
@@ -146,6 +146,7 @@
INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h']
compilation_info = ExternalCompilationInfo(
+ pre_include_lines = ['#define _FILE_OFFSET_BITS 64'],
includes = INCLUDES
)
Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_extfunc.py (original)
+++ pypy/dist/pypy/translator/c/test/test_extfunc.py Wed Feb 27 14:54:38 2008
@@ -111,12 +111,10 @@
os.unlink(filename)
def test_largefile():
- if sys.platform == 'darwin':
- skip("no sparse files on default Mac OS X file system")
if not hasattr(os, 'ftruncate'):
py.test.skip("this os has no ftruncate :-(")
- if os.name == 'nt':
- py.test.skip("no sparse files on Windows")
+ from pypy.module.posix.test.test_posix2 import need_sparse_files
+ need_sparse_files()
filename = str(udir.join('test_largefile'))
r4800000000 = r_longlong(4800000000L)
r4900000000 = r_longlong(4900000000L)
Modified: pypy/dist/pypy/translator/c/test/test_standalone.py
==============================================================================
--- pypy/dist/pypy/translator/c/test/test_standalone.py (original)
+++ pypy/dist/pypy/translator/c/test/test_standalone.py Wed Feb 27 14:54:38 2008
@@ -1,6 +1,7 @@
import py
import sys, os
+from pypy.rlib.rarithmetic import r_longlong
from pypy.translator.translator import TranslationContext
from pypy.translator.backendopt import all
from pypy.translator.c.genc import CStandaloneBuilder
@@ -164,3 +165,28 @@
exe = t.compile()
out = py.process.cmdexec("%s 500" % exe)
assert int(out) == 500*501/2
+
+def test_standalone_large_files():
+ from pypy.module.posix.test.test_posix2 import need_sparse_files
+ need_sparse_files()
+ filename = str(udir.join('test_standalone_largefile'))
+ r4800000000 = r_longlong(4800000000L)
+ def entry_point(argv):
+ fd = os.open(filename, os.O_RDWR | os.O_CREAT, 0644)
+ os.lseek(fd, r4800000000, 0)
+ os.write(fd, "$")
+ newpos = os.lseek(fd, 0, 1)
+ if newpos == r4800000000 + 1:
+ print "OK"
+ else:
+ print "BAD POS"
+ os.close(fd)
+ return 0
+ t = TranslationContext()
+ t.buildannotator().build_types(entry_point, [s_list_of_strings])
+ t.buildrtyper().specialize()
+ cbuilder = CStandaloneBuilder(t, entry_point, t.config)
+ cbuilder.generate_source()
+ cbuilder.compile()
+ data = cbuilder.cmdexec('hi there')
+ assert data.strip() == "OK"
More information about the Pypy-commit
mailing list