[pypy-svn] r45697 - in pypy/branch/pypy-more-rtti-inprogress: module/posix module/posix/test rpython/module translator/c/test
arigo at codespeak.net
arigo at codespeak.net
Thu Aug 16 09:15:32 CEST 2007
Author: arigo
Date: Thu Aug 16 09:15:31 2007
New Revision: 45697
Added:
pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py (contents, props changed)
Modified:
pypy/branch/pypy-more-rtti-inprogress/module/posix/__init__.py
pypy/branch/pypy-more-rtti-inprogress/module/posix/app_posix.py
pypy/branch/pypy-more-rtti-inprogress/module/posix/interp_posix.py
pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py
pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py
pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py
Log:
A more complete support for os.stat(). The RPython one now returns
a Controlled object that supports both tuple-like indexing and the
st_xxx attributes. The extra attributes are then exposed at app-level.
Note that support for sub-second timestamps is there, but possibly a
bit fragile. I disabled it because in CPython 2.4 by default
we don't get sub-second timestamps out of os.stat() either.
Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/__init__.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/module/posix/__init__.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/module/posix/__init__.py Thu Aug 16 09:15:31 2007
@@ -50,6 +50,7 @@
#'getuid' : 'interp_posix.getuid',
#'geteuid' : 'interp_posix.geteuid',
'utime' : 'interp_posix.utime',
+ '_statfields': 'interp_posix.getstatfields(space)',
}
if hasattr(os, 'ftruncate'):
interpleveldefs['ftruncate'] = 'interp_posix.ftruncate'
Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/app_posix.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/module/posix/app_posix.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/module/posix/app_posix.py Thu Aug 16 09:15:31 2007
@@ -2,6 +2,15 @@
from _structseq import structseqtype, structseqfield
+# XXX we need a way to access the current module's globals more directly...
+import sys
+if 'posix' in sys.builtin_module_names:
+ import posix
+elif 'nt' in sys.builtin_module_names:
+ import nt as posix
+else:
+ raise ImportError("XXX")
+
error = OSError
@@ -15,10 +24,28 @@
st_uid = structseqfield(4, "user ID of owner")
st_gid = structseqfield(5, "group ID of owner")
st_size = structseqfield(6, "total size, in bytes")
- st_atime = structseqfield(7, "time of last access (XXX as an int)")
- st_mtime = structseqfield(8, "time of last modification (XXX as an int)")
- st_ctime = structseqfield(9, "time of last change (XXX as an int)")
- # XXX no extra fields for now
+
+ # NOTE: float times are disabled for now, for compatibility with CPython.
+
+ # access to indices 7 to 9 gives the timestamps as integers:
+ #_integer_atime = structseqfield(7)
+ #_integer_mtime = structseqfield(8)
+ #_integer_ctime = structseqfield(9)
+
+ st_atime = structseqfield(7, "time of last access")
+ st_mtime = structseqfield(8, "time of last modification")
+ st_ctime = structseqfield(9, "time of last status change")
+
+ # further fields, not accessible by index (the numbers are still needed
+ # but not visible because they are no longer consecutive)
+ if "st_blksize" in posix._statfields:
+ st_blksize = structseqfield(20, "blocksize for filesystem I/O")
+ if "st_blocks" in posix._statfields:
+ st_blocks = structseqfield(21, "number of blocks allocated")
+ if "st_rdev" in posix._statfields:
+ st_rdev = structseqfield(22, "device ID (if special file)")
+ if "st_flags" in posix._statfields:
+ st_flags = structseqfield(23, "user defined flags for file")
def fdopen(fd, mode='r', buffering=-1):
Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/interp_posix.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/module/posix/interp_posix.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/module/posix/interp_posix.py Thu Aug 16 09:15:31 2007
@@ -1,7 +1,9 @@
from pypy.interpreter.baseobjspace import ObjSpace, W_Root
-from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.unroll import unrolling_iterable
from pypy.interpreter.error import OperationError, wrap_oserror
from pypy.rpython.module.ll_os import RegisterOs
+from pypy.rpython.module import ll_os_stat
+from pypy.rpython.lltypesystem import lltype
import os
@@ -75,14 +77,33 @@
raise wrap_oserror(space, e)
ftruncate.unwrap_spec = [ObjSpace, int, int]
+# ____________________________________________________________
+
+STAT_FIELDS = unrolling_iterable(enumerate(ll_os_stat.STAT_FIELDS))
+
def build_stat_result(space, st):
- # cannot index tuples with a variable...
- lst = [st[0], st[1], st[2], st[3], st[4],
- st[5], st[6], st[7], st[8], st[9]]
- w_tuple = space.newtuple([space.wrap(intmask(x)) for x in lst])
+ lst = []
+ w_keywords = space.newdict()
+ for i, (name, TYPE) in STAT_FIELDS:
+ value = getattr(st, name)
+ #if name in ('st_atime', 'st_mtime', 'st_ctime'):
+ # value = int(value) # rounded to an integer for indexed access
+ w_value = space.wrap(value)
+ if i < ll_os_stat.N_INDEXABLE_FIELDS:
+ lst.append(w_value)
+ else:
+ space.setitem(w_keywords, space.wrap(name), w_value)
+
+ # NOTE: float times are disabled for now, for compatibility with CPython
+ # non-rounded values for name-based access
+ #space.setitem(w_keywords, space.wrap('st_atime'), space.wrap(st.st_atime))
+ #space.setitem(w_keywords, space.wrap('st_mtime'), space.wrap(st.st_mtime))
+ #space.setitem(w_keywords, space.wrap('st_ctime'), space.wrap(st.st_ctime))
+
+ w_tuple = space.newtuple(lst)
w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
space.wrap('stat_result'))
- return space.call_function(w_stat_result, w_tuple)
+ return space.call_function(w_stat_result, w_tuple, w_keywords)
def fstat(space, fd):
"""Perform a stat system call on the file referenced to by an open
@@ -235,10 +256,14 @@
return space.wrap(text)
strerror.unwrap_spec = [ObjSpace, int]
+# ____________________________________________________________
+
+def getstatfields(space):
+ # for app_posix.py: export the list of 'st_xxx' names that we know
+ # about at RPython level
+ return space.newlist(
+ [space.wrap(name) for name in ll_os_stat.STAT_FIELD_NAMES])
-# this is a particular case, because we need to supply
-# the storage for the environment variables, at least
-# for some OSes.
class State:
def __init__(self, space):
Modified: pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/module/posix/test/test_posix2.py Thu Aug 16 09:15:31 2007
@@ -41,11 +41,39 @@
posix.lseek(fd, 5, 0)
s = posix.read(fd, 1)
assert s == 'i'
- stat = posix.fstat(fd)
- assert stat # XXX
+ st = posix.fstat(fd)
posix.close(fd2)
posix.close(fd)
+ import sys, stat
+ assert st[0] == st.st_mode
+ assert st[1] == st.st_ino
+ assert st[2] == st.st_dev
+ assert st[3] == st.st_nlink
+ assert st[4] == st.st_uid
+ assert st[5] == st.st_gid
+ assert st[6] == st.st_size
+ assert st[7] == int(st.st_atime)
+ assert st[8] == int(st.st_mtime)
+ assert st[9] == int(st.st_ctime)
+
+ assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
+ assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
+ if not sys.platform.startswith('win'):
+ assert not (stat.S_IMODE(st.st_mode) & stat.S_IXUSR)
+
+ assert st.st_size == 14
+ assert st.st_nlink == 1
+
+ #if sys.platform.startswith('linux2'):
+ # # expects non-integer timestamps - it's unlikely that they are
+ # # all three integers
+ # assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
+ # (st[7], st[8], st[9]))
+ # assert st.st_blksize * st.st_blocks >= st.st_size
+ if sys.platform.startswith('linux2'):
+ assert hasattr(st, 'st_rdev')
+
def test_pickle(self):
import pickle, os
st = self.posix.stat(os.curdir)
Modified: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os.py Thu Aug 16 09:15:31 2007
@@ -11,7 +11,6 @@
from pypy.tool.sourcetools import func_with_new_name
from pypy.rlib.rarithmetic import r_longlong
from pypy.tool.staticmethods import ClassMethods
-import stat
from pypy.rpython.extfunc import BaseLazyRegistering, registering
from pypy.annotation.model import SomeInteger, SomeString, SomeTuple, SomeFloat
from pypy.annotation.model import s_ImpossibleValue, s_None, s_Bool
@@ -543,82 +542,20 @@
# --------------------------- os.stat & variants ---------------------------
- def register_stat_variant(self, name):
- if sys.platform.startswith('win'):
- struct_stat = '_stati64'
- functions = {'stat': '_stati64',
- 'fstat': '_fstati64',
- 'lstat': '_stati64'} # no lstat on Windows
- c_func_name = functions[name]
- INCLUDES = []
- else:
- struct_stat = 'stat'
- c_func_name = name
- INCLUDES = self.UNISTD_INCL + ['sys/stat.h']
- # XXX all fields are lltype.Signed for now, which is wrong
- STRUCT_STAT = rffi.CStruct(struct_stat,
- ('st_mode', lltype.Signed),
- ('st_ino', lltype.Signed),
- ('st_dev', lltype.Signed),
- ('st_nlink', lltype.Signed),
- ('st_uid', lltype.Signed),
- ('st_gid', lltype.Signed),
- ('st_size', lltype.Signed),
- ('st_atime', lltype.Signed),
- ('st_mtime', lltype.Signed),
- ('st_ctime', lltype.Signed),
- )
- arg_is_path = (name != 'fstat')
- if arg_is_path:
- ARG1 = rffi.CCHARP
- else:
- ARG1 = rffi.INT
- os_mystat = rffi.llexternal(name, [ARG1, STRUCT_STAT], rffi.INT,
- includes=INCLUDES)
-
- def os_mystat_lltypeimpl(arg):
- stresult = lltype.malloc(STRUCT_STAT.TO, flavor='raw')
- try:
- if arg_is_path:
- arg = rffi.str2charp(arg)
- error = os_mystat(arg, stresult)
- if arg_is_path:
- rffi.free_charp(arg)
- if error != 0:
- raise OSError(rffi.get_errno(), "os_stat failed")
- return (stresult.c_st_mode,
- stresult.c_st_ino,
- stresult.c_st_dev,
- stresult.c_st_nlink,
- stresult.c_st_uid,
- stresult.c_st_gid,
- stresult.c_st_size,
- stresult.c_st_atime,
- stresult.c_st_mtime,
- stresult.c_st_ctime)
- finally:
- lltype.free(stresult, flavor='raw')
-
- if arg_is_path:
- s_arg = str
- else:
- s_arg = int
- self.register(getattr(os, name), [s_arg], (int,) * 10,
- "ll_os.ll_os_%s" % (name,),
- llimpl=func_with_new_name(os_mystat_lltypeimpl,
- 'os_%s_lltypeimpl' % (name,)))
-
@registering(os.fstat)
def register_os_fstat(self):
- self.register_stat_variant('fstat')
+ from pypy.rpython.module import ll_os_stat
+ ll_os_stat.register_stat_variant('fstat')
@registering(os.stat)
def register_os_stat(self):
- self.register_stat_variant('stat')
+ from pypy.rpython.module import ll_os_stat
+ ll_os_stat.register_stat_variant('stat')
@registering(os.lstat)
def register_os_lstat(self):
- self.register_stat_variant('lstat')
+ from pypy.rpython.module import ll_os_stat
+ ll_os_stat.register_stat_variant('lstat')
# ------------------------------- os.W* ---------------------------------
Added: pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py
==============================================================================
--- (empty file)
+++ pypy/branch/pypy-more-rtti-inprogress/rpython/module/ll_os_stat.py Thu Aug 16 09:15:31 2007
@@ -0,0 +1,192 @@
+"""Annotation and rtyping support for the result of os.stat(), os.lstat()
+and os.fstat(). In RPython like in plain Python the stat result can be
+indexed like a tuple but also exposes the st_xxx attributes.
+"""
+import os, sys
+from pypy.annotation import model as annmodel
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.controllerentry import Controller, SomeControlledInstance
+from pypy.rpython.extfunc import _register_external
+from pypy.rpython.lltypesystem import rffi, lltype
+
+if sys.platform.startswith('win'):
+ # XXX on Windows, stat() is flawed; see CPython's posixmodule.c for
+ # an implementation based on the Win32 API
+ LongLongIfNotWindows = lltype.Signed
+else:
+ LongLongIfNotWindows = lltype.SignedLongLong
+
+# NOTE: float times are disabled for now, for compatibility with CPython
+if 0: #sys.platform.startswith('linux2'):
+ # XXX assume the tv_nsec way of accessing the sub-second timestamps
+ # XXX also assume, as in Linux, that it's in a 'struct timespec'
+ TIMESPEC = rffi.CStruct('timespec',
+ ('tv_sec', lltype.Signed),
+ ('tv_nsec', lltype.Signed))
+ ModTime = rffi.DOUBLE
+else:
+ # XXX add support for more platforms
+ TIMESPEC = None
+ ModTime = lltype.Signed
+
+# all possible fields - some of them are not available on all platforms
+ALL_STAT_FIELDS = [
+ ("st_mode", lltype.Signed),
+ ("st_ino", lltype.SignedLongLong),
+ ("st_dev", LongLongIfNotWindows),
+ ("st_nlink", lltype.Signed),
+ ("st_uid", lltype.Signed),
+ ("st_gid", lltype.Signed),
+ ("st_size", lltype.SignedLongLong),
+ ("st_atime", ModTime),
+ ("st_mtime", ModTime),
+ ("st_ctime", ModTime),
+ ("st_blksize", lltype.Signed),
+ ("st_blocks", lltype.Signed),
+ ("st_rdev", lltype.Signed),
+ ("st_flags", lltype.Signed),
+ #("st_gen", lltype.Signed), -- new in CPy 2.5, not implemented
+ #("st_birthtime", ModTime), -- new in CPy 2.5, not implemented
+ ]
+N_INDEXABLE_FIELDS = 10
+
+# for now, check the host Python to know which st_xxx fields exist
+STAT_FIELDS = [(_name, _TYPE) for (_name, _TYPE) in ALL_STAT_FIELDS
+ if hasattr(os.stat_result, _name)]
+
+STAT_FIELD_TYPES = dict(STAT_FIELDS) # {'st_xxx': TYPE}
+
+STAT_FIELD_NAMES = [_name for (_name, _TYPE) in ALL_STAT_FIELDS
+ if _name in STAT_FIELD_TYPES]
+
+def _expand(lst, originalname, timespecname):
+ if TIMESPEC is not None:
+ for i, (_name, _TYPE) in enumerate(lst):
+ if _name == originalname:
+ # replace the 'st_atime' field of type rffi.DOUBLE
+ # with a field 'st_atim' of type 'struct timespec'
+ lst[i] = (timespecname, TIMESPEC.TO)
+ break
+
+LL_STAT_FIELDS = STAT_FIELDS[:]
+_expand(LL_STAT_FIELDS, 'st_atime', 'st_atim')
+_expand(LL_STAT_FIELDS, 'st_mtime', 'st_mtim')
+_expand(LL_STAT_FIELDS, 'st_ctime', 'st_ctim')
+
+del _expand, _name, _TYPE
+
+# ____________________________________________________________
+#
+# Annotation support
+
+class StatResultController(Controller):
+ """Controls a stat_result object in RPython: internally it is just a
+ tuple, but the Controller adds the support for the st_xxx attributes.
+ """
+ knowntype = os.stat_result
+
+ def getitem(self, obj, index):
+ if 0 <= index < N_INDEXABLE_FIELDS:
+ return obj[index]
+ else:
+ raise IndexError
+ getitem._annspecialcase_ = 'specialize:arg(2)' # 'index' must be constant
+
+ def install_getter(cls, name):
+ # def get_st_mode(), def get_st_ino(), etc...
+ index = STAT_FIELD_NAMES.index(name)
+ def get_st_xxx(self, obj):
+ return obj[index]
+ method_name = 'get_%s' % (name,)
+ setattr(cls, method_name, func_with_new_name(get_st_xxx, method_name))
+ install_getter = classmethod(install_getter)
+
+for _name in STAT_FIELD_NAMES:
+ StatResultController.install_getter(_name)
+
+stat_controller = StatResultController()
+s_tuple_StatResult = annmodel.SomeTuple([annmodel.lltype_to_annotation(_TYPE)
+ for _name, _TYPE in STAT_FIELDS])
+s_StatResult = SomeControlledInstance(s_tuple_StatResult,
+ controller = stat_controller)
+
+# ____________________________________________________________
+#
+# RFFI support
+
+if sys.platform.startswith('win'):
+ _name_struct_stat = '_stati64'
+ INCLUDES = []
+else:
+ _name_struct_stat = 'stat'
+ INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h']
+STRUCT_STAT = rffi.CStruct(_name_struct_stat, *LL_STAT_FIELDS)
+
+
+def build_stat_result(st):
+ if TIMESPEC is not None:
+ atim = st.c_st_atim; atime = atim.c_tv_sec + 1E-9 * atim.c_tv_nsec
+ mtim = st.c_st_mtim; mtime = mtim.c_tv_sec + 1E-9 * mtim.c_tv_nsec
+ ctim = st.c_st_ctim; ctime = ctim.c_tv_sec + 1E-9 * ctim.c_tv_nsec
+ else:
+ atime = st.c_st_atime
+ mtime = st.c_st_mtime
+ ctime = st.c_st_ctime
+
+ result = (st.c_st_mode,
+ st.c_st_ino,
+ st.c_st_dev,
+ st.c_st_nlink,
+ st.c_st_uid,
+ st.c_st_gid,
+ st.c_st_size,
+ atime,
+ mtime,
+ ctime)
+
+ if "st_blksize" in STAT_FIELD_TYPES: result += (st.c_st_blksize,)
+ if "st_blocks" in STAT_FIELD_TYPES: result += (st.c_st_blocks,)
+ if "st_rdev" in STAT_FIELD_TYPES: result += (st.c_st_rdev,)
+ if "st_flags" in STAT_FIELD_TYPES: result += (st.c_st_flags,)
+
+ return stat_controller.box(result)
+
+
+def register_stat_variant(name):
+ if sys.platform.startswith('win'):
+ _functions = {'stat': '_stati64',
+ 'fstat': '_fstati64',
+ 'lstat': '_stati64'} # no lstat on Windows
+ c_func_name = _functions[name]
+ else:
+ c_func_name = name
+ arg_is_path = (name != 'fstat')
+ if arg_is_path:
+ ARG1 = rffi.CCHARP
+ else:
+ ARG1 = rffi.INT
+ os_mystat = rffi.llexternal(name, [ARG1, STRUCT_STAT], rffi.INT,
+ includes=INCLUDES)
+
+ def os_mystat_lltypeimpl(arg):
+ stresult = lltype.malloc(STRUCT_STAT.TO, flavor='raw')
+ try:
+ if arg_is_path:
+ arg = rffi.str2charp(arg)
+ error = os_mystat(arg, stresult)
+ if arg_is_path:
+ rffi.free_charp(arg)
+ if error != 0:
+ raise OSError(rffi.get_errno(), "os_?stat failed")
+ return build_stat_result(stresult)
+ finally:
+ lltype.free(stresult, flavor='raw')
+
+ if arg_is_path:
+ s_arg = str
+ else:
+ s_arg = int
+ _register_external(getattr(os, name), [s_arg], s_StatResult,
+ "ll_os.ll_os_%s" % (name,),
+ llimpl=func_with_new_name(os_mystat_lltypeimpl,
+ 'os_%s_lltypeimpl' % (name,)))
Modified: pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py
==============================================================================
--- pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py (original)
+++ pypy/branch/pypy-more-rtti-inprogress/translator/c/test/test_extfunc.py Thu Aug 16 09:15:31 2007
@@ -132,14 +132,27 @@
def test_os_stat():
filename = str(py.magic.autopath())
+ has_blksize = hasattr(os.stat_result, 'st_blksize')
+ has_blocks = hasattr(os.stat_result, 'st_blocks')
def call_stat():
st = os.stat(filename)
- return st
+ res = (st[0], st.st_ino, st.st_ctime)
+ if has_blksize: res += (st.st_blksize,)
+ if has_blocks: res += (st.st_blocks,)
+ return res
f = compile(call_stat, [])
- result = f()
- assert result[0] == os.stat(filename)[0]
- assert result[1] == os.stat(filename)[1]
- assert result[2] == os.stat(filename)[2]
+ res = f()
+ assert res[0] == os.stat(filename).st_mode
+ assert res[1] == os.stat(filename).st_ino
+ st_ctime = res[2]
+ if isinstance(st_ctime, float):
+ assert st_ctime == os.stat(filename).st_ctime
+ else:
+ assert st_ctime == int(os.stat(filename).st_ctime)
+ if has_blksize:
+ assert res[3] == os.stat(filename).st_blksize
+ if has_blocks:
+ assert res[4] == os.stat(filename).st_blocks
def test_os_fstat():
if os.environ.get('PYPY_CC', '').startswith('tcc'):
@@ -148,18 +161,17 @@
fd = os.open(filename, os.O_RDONLY, 0777)
def call_fstat(fd):
st = os.fstat(fd)
- return st
+ return (st.st_mode, st[1], st.st_mtime)
f = compile(call_fstat, [int])
osstat = os.stat(filename)
- result = f(fd)
+ st_mode, st_ino, st_mtime = f(fd)
os.close(fd)
- import stat
- for i in range(len(result)):
- if i == stat.ST_DEV:
- continue # does give 3 instead of 0 for windows
- elif i == stat.ST_ATIME:
- continue # access time will vary
- assert (i, result[i]) == (i, osstat[i])
+ assert st_mode == osstat.st_mode
+ assert st_ino == osstat.st_ino
+ if isinstance(st_mtime, float):
+ assert st_mtime == osstat.st_mtime
+ else:
+ assert st_mtime == int(osstat.st_mtime)
def test_os_isatty():
def call_isatty(fd):
More information about the Pypy-commit
mailing list