[pypy-svn] r79223 - in pypy/branch/fast-forward/pypy/module/_io: . test
afa at codespeak.net
afa at codespeak.net
Wed Nov 17 20:02:24 CET 2010
Author: afa
Date: Wed Nov 17 20:02:23 2010
New Revision: 79223
Modified:
pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py
pypy/branch/fast-forward/pypy/module/_io/interp_bytesio.py
pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py
Log:
Have BufferedReader.read() used raw.readinto just like CPython.
Implement a true BytesIO.readinto()
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_bufferedio.py Wed Nov 17 20:02:23 2010
@@ -4,9 +4,10 @@
from pypy.interpreter.gateway import interp2app, unwrap_spec, Arguments
from pypy.interpreter.baseobjspace import ObjSpace, W_Root
from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.buffer import RWBuffer
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.rlib.rstring import StringBuilder
-from pypy.rlib.rarithmetic import r_longlong
+from pypy.rlib.rarithmetic import r_longlong, intmask
from pypy.tool.sourcetools import func_renamer
from pypy.module._io.interp_iobase import (
W_IOBase, convert_size,
@@ -71,6 +72,17 @@
readinto = interp2app(W_BufferedIOBase.readinto_w),
)
+class RawBuffer(RWBuffer):
+ def __init__(self, buf, length):
+ self.buf = buf
+ self.length = length
+
+ def getlength(self):
+ return self.length
+
+ def setitem(self, index, char):
+ self.buf[index] = char
+
class BufferedMixin:
_mixin_ = True
@@ -480,25 +492,24 @@
self.abs_pos += size
return builder.build()
- def _raw_read(self, space, n):
- w_data = space.call_method(self.w_raw, "read", space.wrap(n))
- if space.is_w(w_data, space.w_None):
+ def _raw_read(self, space, buffer, start, length):
+ length = intmask(length)
+ w_buf = space.wrap(RawBuffer(rffi.ptradd(buffer, start), length))
+ w_size = space.call_method(self.w_raw, "readinto", w_buf)
+ if space.is_w(w_size, space.w_None):
raise BlockingIOError()
- data = space.str_w(w_data)
+ size = space.int_w(w_size)
if self.abs_pos != -1:
- self.abs_pos += len(data)
- return data
+ self.abs_pos += size
+ return size
def _fill_buffer(self, space):
start = self.read_end
if start == -1:
start = 0
length = self.buffer_size - start
- data = self._raw_read(space, length)
- size = len(data)
+ size = self._raw_read(space, self.buffer, start, length)
if size > 0:
- for i in range(size):
- self.buffer[start + i] = data[i]
self.read_end = self.raw_pos = start + size
return size
@@ -509,70 +520,64 @@
if n <= current_size:
return self._read_fast(n)
- builder = StringBuilder(n)
- remaining = n
- written = 0
- data = None
- if current_size:
- data = rffi.charpsize2str(rffi.ptradd(self.buffer, self.pos),
- current_size)
- builder.append(data)
- remaining -= len(data)
- written += len(data)
- self._reader_reset_buf()
+ with rffi.scoped_alloc_buffer(n) as result_buffer:
+ remaining = n
+ written = 0
+ data = None
+ if current_size:
+ for i in range(current_size):
+ result_buffer.raw[written + i] = self.buffer[self.pos + i]
+ remaining -= current_size
+ written += current_size
+ self._reader_reset_buf()
- # XXX potential bug in CPython? The following is not enabled.
- # We're going past the buffer's bounds, flush it
- ## if self.writable:
- ## self._writer_flush_unlocked(space, restore_pos=True)
+ # XXX potential bug in CPython? The following is not enabled.
+ # We're going past the buffer's bounds, flush it
+ ## if self.writable:
+ ## self._writer_flush_unlocked(space, restore_pos=True)
+
+ # Read whole blocks, and don't buffer them
+ while remaining > 0:
+ r = self.buffer_size * (remaining // self.buffer_size)
+ if r == 0:
+ break
+ try:
+ size = self._raw_read(space, result_buffer.raw, written, r)
+ except BlockingIOError:
+ if written == 0:
+ return None
+ size = 0
+ if size == 0:
+ return result_buffer.str(intmask(written))
+ remaining -= size
+ written += size
- # Read whole blocks, and don't buffer them
- while remaining > 0:
- r = self.buffer_size * (remaining // self.buffer_size)
- if r == 0:
- break
- try:
- data = self._raw_read(space, r)
- except BlockingIOError:
- if written == 0:
- return None
- data = ""
- size = len(data)
- if size == 0:
- return builder.build()
- builder.append(data)
- remaining -= size
- written += size
+ self.pos = 0
+ self.raw_pos = 0
+ self.read_end = 0
- self.pos = 0
- self.raw_pos = 0
- self.read_end = 0
+ while remaining > 0 and self.read_end < self.buffer_size:
+ try:
+ size = self._fill_buffer(space)
+ except BlockingIOError:
+ # EOF or read() would block
+ if written == 0:
+ return None
+ size = 0
+ if size == 0:
+ break
- while remaining > 0 and self.read_end < self.buffer_size:
- try:
- size = self._fill_buffer(space)
- except BlockingIOError:
- # EOF or read() would block
- if written == 0:
- return None
- size = 0
- if size == 0:
- break
+ if remaining > 0:
+ if size > remaining:
+ size = remaining
+ for i in range(size):
+ result_buffer.raw[written + i] = self.buffer[self.pos + i]
+ self.pos += size
- if remaining > 0:
- if size > remaining:
- size = remaining
- # XXX inefficient
- l = []
- for i in range(self.pos,self.pos + size):
- l.append(self.buffer[i])
- data = ''.join(l)
- builder.append(data)
+ written += size
+ remaining -= size
- written += size
- self.pos += size
- remaining -= size
- return builder.build()
+ return result_buffer.str(intmask(written))
def _read_fast(self, n):
"""Read n bytes from the buffer if it can, otherwise return None.
Modified: pypy/branch/fast-forward/pypy/module/_io/interp_bytesio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/interp_bytesio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/interp_bytesio.py Wed Nov 17 20:02:23 2010
@@ -49,6 +49,20 @@
return space.wrap(output)
@unwrap_spec('self', ObjSpace, W_Root)
+ def readinto_w(self, space, w_buffer):
+ rwbuffer = space.rwbuffer_w(w_buffer)
+ size = rwbuffer.getlength()
+
+ if self.pos + size > self.string_size:
+ size = self.string_size - self.pos
+
+ output = buffer2string(self.buf, self.pos, self.pos + size)
+ length = len(output)
+ rwbuffer.setslice(0, output)
+ self.pos += length
+ return space.wrap(length)
+
+ @unwrap_spec('self', ObjSpace, W_Root)
def write_w(self, space, w_data):
self._check_closed(space)
buf = space.buffer_w(w_data)
@@ -157,6 +171,7 @@
__init__ = interp2app(W_BytesIO.descr_init),
read = interp2app(W_BytesIO.read_w),
+ readinto = interp2app(W_BytesIO.readinto_w),
write = interp2app(W_BytesIO.write_w),
truncate = interp2app(W_BytesIO.truncate_w),
getvalue = interp2app(W_BytesIO.getvalue_w),
Modified: pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py
==============================================================================
--- pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py (original)
+++ pypy/branch/fast-forward/pypy/module/_io/test/test_bufferedio.py Wed Nov 17 20:02:23 2010
@@ -37,15 +37,65 @@
class MockIO(_io._IOBase):
def readable(self):
return True
- def read(self, n=-1): # PyPy uses read()
- return "abc"
- def readinto(self, buf): # CPython uses readinto()
+ def readinto(self, buf):
buf[:3] = "abc"
return 3
bufio = _io.BufferedReader(MockIO())
r = bufio.read(5)
assert r == "abcab"
+ def test_read_past_eof(self):
+ import _io
+ class MockIO(_io._IOBase):
+ stack = ["abc", "d", "efg"]
+ def readable(self):
+ return True
+ def readinto(self, buf):
+ if self.stack:
+ data = self.stack.pop(0)
+ buf[:len(data)] = data
+ return len(data)
+ else:
+ return 0
+ bufio = _io.BufferedReader(MockIO())
+ assert bufio.read(9000) == "abcdefg"
+
+ def test_buffering(self):
+ import _io
+ data = b"abcdefghi"
+ dlen = len(data)
+ class MockFileIO(_io.BytesIO):
+ def __init__(self, data):
+ self.read_history = []
+ _io.BytesIO.__init__(self, data)
+
+ def read(self, n=None):
+ res = _io.BytesIO.read(self, n)
+ self.read_history.append(None if res is None else len(res))
+ return res
+
+ def readinto(self, b):
+ res = _io.BytesIO.readinto(self, b)
+ self.read_history.append(res)
+ return res
+
+
+ tests = [
+ [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ],
+ [ 100, [ 3, 3, 3], [ dlen ] ],
+ [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ],
+ ]
+
+ for bufsize, buf_read_sizes, raw_read_sizes in tests:
+ rawio = MockFileIO(data)
+ bufio = _io.BufferedReader(rawio, buffer_size=bufsize)
+ pos = 0
+ for nbytes in buf_read_sizes:
+ assert bufio.read(nbytes) == data[pos:pos+nbytes]
+ pos += nbytes
+ # this is mildly implementation-dependent
+ assert rawio.read_history == raw_read_sizes
+
def test_peek(self):
import _io
raw = _io.FileIO(self.tmpfile)
More information about the Pypy-commit
mailing list