[pypy-svn] r78257 - in pypy/branch/fast-forward/pypy/module: _io _io/test thread
afa at codespeak.net
afa at codespeak.net
Mon Oct 25 14:17:00 CEST 2010
Author: afa
Date: Mon Oct 25 14:16:59 2010
New Revision: 78257
Modified:
pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py
pypy/branch/fast-forward/pypy/module/_io/interp_fileio.py
pypy/branch/fast-forward/pypy/module/_io/interp_iobase.py
pypy/branch/fast-forward/pypy/module/_io/interp_stringio.py
pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py
pypy/branch/fast-forward/pypy/module/thread/os_lock.py
Log:
Implement BufferedReader.read()
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py Mon Oct 25 14:16:59 2010
@@ -1,10 +1,12 @@
+from __future__ import with_statement
from pypy.interpreter.typedef import (
TypeDef, generic_new_descr)
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.baseobjspace import ObjSpace, W_Root
from pypy.interpreter.error import OperationError
from pypy.rpython.lltypesystem import lltype, rffi
-from pypy.module._io.interp_iobase import W_IOBase
+from pypy.rlib.rstring import StringBuilder
+from pypy.module._io.interp_iobase import W_IOBase, convert_size
from pypy.module._io.interp_io import DEFAULT_BUFFER_SIZE
from pypy.module.thread.os_lock import Lock
@@ -14,6 +16,9 @@
self.buffer = lltype.nullptr(rffi.CCHARP.TO)
self.lock = None
+ self.readable = False
+ self.writable = False
+
def _init(self, space):
if self.buffer_size <= 0:
raise OperationError(space.w_ValueError, space.wrap(
@@ -44,9 +49,24 @@
self.abs_pos = pos
return pos
+ def _readahead(self):
+ if self.readable and self.read_end != -1:
+ return self.read_end - self.pos
+ return 0
+
+ def _unsupportedoperation(self, space, message):
+ w_exc = space.getattr(space.getbuiltinmodule('_io'),
+ space.wrap('UnsupportedOperation'))
+ raise OperationError(w_exc, space.wrap(message))
+
+ @unwrap_spec('self', ObjSpace, W_Root)
+ def read_w(self, space, w_size=None):
+ self._unsupportedoperation(space, "read")
+
W_BufferedIOBase.typedef = TypeDef(
'_BufferedIOBase', W_IOBase.typedef,
__new__ = generic_new_descr(W_BufferedIOBase),
+ read = interp2app(W_BufferedIOBase.read_w),
)
class W_BufferedReader(W_BufferedIOBase):
@@ -63,7 +83,6 @@
self.raw = raw
self.buffer_size = buffer_size
self.readable = True
- self.writable = False
self._init(space)
self._reset_buf()
@@ -71,10 +90,108 @@
def _reset_buf(self):
self.read_end = -1
+ def _closed(self, space):
+ return self.raw._closed(space)
+
+ @unwrap_spec('self', ObjSpace, W_Root)
+ def read_w(self, space, w_size=None):
+ self._check_closed(space, "read of closed file")
+ size = convert_size(space, w_size)
+
+ if size < 0:
+ # read until the end of stream
+ with self.lock:
+ res = self._read_all(space)
+ else:
+ res = self._read_fast(size)
+ if res is None:
+ with self.lock:
+ res = self._read_generic(space, size)
+ return space.wrap(res)
+
+ def _read_all(self, space):
+ builder = StringBuilder()
+ # First copy what we have in the current buffer
+ current_size = self._readahead()
+ data = None
+ if current_size:
+ data = rffi.charpsize2str(rffi.ptradd(self.buffer, self.pos),
+ current_size)
+ builder.append(data)
+ self._reset_buf()
+ # We're going past the buffer's bounds, flush it
+ if self.writable:
+ self._writer_flush_unlocked(restore_pos=True)
+
+ while True:
+ # Read until EOF or until read() would block
+ w_data = space.call_method(self.raw, "read")
+ if space.is_w(w_data, space.w_None):
+ break
+ data = space.str_w(w_data)
+ size = len(data)
+ if size == 0:
+ break
+ builder.append(data)
+ current_size += size
+ if self.abs_pos != -1:
+ self.abs_pos += size
+ return builder.build()
+
+ def _read_generic(self, space, n):
+ """Generic read function: read from the stream until enough bytes are
+ read, or until an EOF occurs or until read() would block."""
+ current_size = self._readahead()
+ if n <= current_size:
+ return self._read_fast(n)
+
+ builder = StringBuilder(n)
+ remaining = n
+ written = 0
+ data = None
+ if current_size:
+ data = rffi.charpsize2str(rffi.ptradd(self.buffer, self.pos),
+ current_size)
+ builder.append(data)
+ self._reset_buf()
+
+ # XXX potential bug in CPython? The following is not enabled.
+ # We're going past the buffer's bounds, flush it
+ ## if self.writable:
+ ## self._writer_flush_unlocked(restore_pos=True)
+
+ while remaining > 0:
+ # Read until EOF or until read() would block
+ w_data = space.call_method(self.raw, "read", space.wrap(remaining))
+ if space.is_w(w_data, space.w_None):
+ break
+ data = space.str_w(w_data)
+ size = len(data)
+ if size == 0:
+ break
+ builder.append(data)
+ current_size += size
+ remaining -= size
+ if self.abs_pos != -1:
+ self.abs_pos += size
+ return builder.build()
+
+ def _read_fast(self, n):
+ """Read n bytes from the buffer if it can, otherwise return None.
+ This function is simple enough that it can run unlocked."""
+ current_size = self._readahead()
+ if n <= current_size:
+ res = rffi.charpsize2str(rffi.ptradd(self.buffer, self.pos), n)
+ self.pos += n
+ return res
+ return None
+
W_BufferedReader.typedef = TypeDef(
'BufferedReader', W_BufferedIOBase.typedef,
__new__ = generic_new_descr(W_BufferedReader),
__init__ = interp2app(W_BufferedReader.descr_init),
+
+ read = interp2app(W_BufferedReader.read_w),
)
class W_BufferedWriter(W_BufferedIOBase):
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_fileio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_fileio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_fileio.py Mon Oct 25 14:16:59 2010
@@ -1,4 +1,3 @@
-from pypy.module._io.interp_iobase import W_RawIOBase
from pypy.interpreter.typedef import (
TypeDef, interp_attrproperty, interp_attrproperty_w, GetSetProperty,
make_weakref_descr)
@@ -9,6 +8,7 @@
from pypy.rlib.rstring import StringBuilder
from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC
import sys, os, stat, errno
+from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
def interp_member_w(name, cls, doc=None):
"NOT_RPYTHON: initialization-time only"
@@ -93,12 +93,6 @@
return readable, writable, append, flags
-def convert_size(space, w_size):
- if space.is_w(w_size, space.w_None):
- return -1
- else:
- return space.int_w(w_size)
-
SMALLCHUNK = 8 * 1024
BIGCHUNK = 512 * 1024
@@ -211,10 +205,14 @@
def descr_get_mode(space, self):
return space.wrap(self._mode())
- def _check_closed(self, space):
+ def _closed(self, space):
+ return self.fd < 0
+
+ def _check_closed(self, space, message=None):
+ if message is None:
+ message = "I/O operation on closed file"
if self.fd < 0:
- raise OperationError(space.w_ValueError, space.wrap(
- "I/O operation on closed file"))
+ raise OperationError(space.w_ValueError, space.wrap(message))
def _close(self, space):
if self.fd < 0:
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_iobase.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_iobase.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_iobase.py Mon Oct 25 14:16:59 2010
@@ -47,11 +47,12 @@
# attribute as returned by whatever subclass.
return self.__IOBase_closed
- def _check_closed(self, space):
+ def _check_closed(self, space, message=None):
+ if message is None:
+ message = "I/O operation on closed file"
if self._closed(space):
raise OperationError(
- space.w_ValueError,
- space.wrap("I/O operation on closed file"))
+ space.w_ValueError, space.wrap(message))
def closed_get_w(space, self):
return space.newbool(self.__IOBase_closed)
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_stringio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_stringio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_stringio.py Mon Oct 25 14:16:59 2010
@@ -9,7 +9,7 @@
self.buf = []
self.pos = 0
- def _check_closed(self, space):
+ def _check_closed(self, space, message=None):
pass
def _check_initialized(self):
pass
Modified: pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py Mon Oct 25 14:16:59 2010
@@ -8,7 +8,15 @@
tmpfile.write("a\nb\nc", mode='wb')
cls.w_tmpfile = cls.space.wrap(str(tmpfile))
- def test_simple(self):
+ def test_simple_read(self):
import _io
raw = _io.FileIO(self.tmpfile)
- _io.BufferedReader(raw)
+ f = _io.BufferedReader(raw)
+ assert f.read() == "a\nb\nc"
+ f.close()
+ #
+ raw.seek(0)
+ f = _io.BufferedReader(raw)
+ r = f.read(4)
+ assert r == "a\nb\n"
+ f.close()
Modified: pypy/branch/fast-forward/pypy/module/thread/os_lock.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/thread/os_lock.py (original)
+++ pypy/branch/fast-forward/pypy/module/thread/os_lock.py Mon Oct 25 14:16:59 2010
@@ -30,6 +30,7 @@
"A wrappable box around an interp-level lock object."
def __init__(self, space):
+ self.space = space
try:
self.lock = thread.allocate_lock()
except thread.error:
@@ -70,6 +71,13 @@
def descr__exit__(self, space, __args__):
self.descr_lock_release(space)
+ def __enter__(self):
+ self.descr_lock_acquire(self.space)
+ return self
+
+ def __exit__(self, *args):
+ self.descr_lock_release(self.space)
+
descr_acquire = interp2app(Lock.descr_lock_acquire,
unwrap_spec=['self', ObjSpace, int])
descr_release = interp2app(Lock.descr_lock_release,
More information about the Pypy-commit
mailing list