Author: antoine.pitrou
Date: Wed Dec 31 00:38:13 2008
New Revision: 68086
Log:
Rewritten BufferedWriter with a static buffer, and misc changes.
Modified:
sandbox/trunk/io-c/_bufferedio.c
sandbox/trunk/io-c/_iobase.c
sandbox/trunk/io-c/_iomodule.h
sandbox/trunk/io-c/build.py
sandbox/trunk/io-c/io.c
sandbox/trunk/io-c/test_io.py
Modified: sandbox/trunk/io-c/_bufferedio.c
==============================================================================
--- sandbox/trunk/io-c/_bufferedio.c (original)
+++ sandbox/trunk/io-c/_bufferedio.c Wed Dec 31 00:38:13 2008
@@ -109,8 +109,14 @@
PyObject *raw;
int ok; /* Initialized? */
- PyObject *read_buf;
+ /* A static buffer of size `buffer_size` */
+ unsigned char *read_buf;
+ /* Current reading pos in the buffer. */
Py_ssize_t read_pos;
+ /* Just after the last buffered byte in the buffer, or -1 if the buffer
+ is invalid.
+ Should most often be equal to buffer_size for "normal" blocking I/O. */
+ Py_ssize_t read_end;
PyThread_type_lock read_lock;
/* A static buffer of size `buffer_size` */
@@ -122,8 +128,6 @@
PyThread_type_lock write_lock;
Py_ssize_t buffer_size;
- /* Still unused, but will help us align reads and writes on block
- boundaries if desireable */
Py_ssize_t buffer_mask;
PyObject *dict;
@@ -162,13 +166,28 @@
return -1; \
}
+#define VALID_READ_BUFFER(self) \
+ (self->read_end != -1)
+
+#define AVAILABLE_BYTES(self) \
+ (VALID_READ_BUFFER(self) ? (self->read_end - self->read_pos) : 0)
+
+#define MINUS_LAST_BLOCK(self, size) \
+ (self->buffer_mask ? \
+ (size & ~self->buffer_mask) : \
+ (self->buffer_size * (size / self->buffer_size)))
+
+
static void
BufferedObject_dealloc(BufferedObject *self)
{
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *)self);
Py_CLEAR(self->raw);
- Py_CLEAR(self->read_buf);
+ if (self->read_buf) {
+ PyMem_Free(self->read_buf);
+ self->write_buf = NULL;
+ }
if (self->write_buf) {
PyMem_Free(self->write_buf);
self->write_buf = NULL;
@@ -331,8 +350,12 @@
return PyObject_CallMethod(self->raw, "isatty", NULL);
}
+/*
+ * Helpers
+ */
+
static int
-_BufferedIOMixin_init(BufferedObject *self)
+_Buffered_init(BufferedObject *self)
{
Py_ssize_t n;
if (self->buffer_size <= 0) {
@@ -350,6 +373,40 @@
return 0;
}
+/* Returns the address of the `written` member if a BlockingIOError was
+ raised, NULL otherwise. The error is always re-raised. */
+static Py_ssize_t *
+_Buffered_check_blocking_error(void)
+{
+ PyObject *t, *v, *tb;
+ PyBlockingIOErrorObject *err;
+
+ PyErr_Fetch(&t, &v, &tb);
+ if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) {
+ PyErr_Restore(t, v, tb);
+ return NULL;
+ }
+ err = (PyBlockingIOErrorObject *) v;
+ /* TODO: sanity check (err->written >= 0) */
+ PyErr_Restore(t, v, tb);
+ return &err->written;
+}
+
+static Py_ssize_t
+_Buffered_raw_tell(BufferedObject *self)
+{
+ PyObject *res;
+ Py_ssize_t n;
+ res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_tell, NULL);
+ if (res == NULL)
+ return -1;
+ n = PyNumber_AsSsize_t(res, PyExc_ValueError);
+ Py_DECREF(res);
+ /* TODO: sanity check (n >= 0) */
+ return n;
+}
+
+
/*
* class BufferedReader
*/
@@ -359,16 +416,8 @@
static int _BufferedReader_reset_buf(BufferedObject *self)
{
- PyObject *oldbuf = self->read_buf;
-
- self->read_buf = PyBytes_FromStringAndSize(NULL, 0);
+ self->read_end = -1;
self->read_pos = 0;
-
- Py_XDECREF(oldbuf);
-
- if (self->read_buf == NULL)
- return -1;
-
return 0;
}
@@ -392,12 +441,19 @@
Py_CLEAR(self->raw);
Py_INCREF(raw);
self->raw = raw;
-
self->buffer_size = buffer_size;
- self->read_buf = NULL;
- if (_BufferedIOMixin_init(self) < 0)
+ if (_Buffered_init(self) < 0)
+ return -1;
+
+ if (self->read_buf)
+ PyMem_Free(self->read_buf);
+ self->read_buf = PyMem_Malloc(self->buffer_size);
+ if (self->read_buf == NULL) {
+ PyErr_NoMemory();
return -1;
+ }
+
if (_BufferedReader_reset_buf(self) < 0)
return -1;
@@ -411,13 +467,39 @@
return 0;
}
+static Py_ssize_t
+_BufferedReader_raw_read(BufferedObject *self, unsigned char *start, Py_ssize_t len)
+{
+ Py_buffer buf;
+ PyObject *memobj, *res;
+ Py_ssize_t n;
+ /* NOTE: the buffer needn't be released as its object is NULL. */
+ if (PyBuffer_FillInfo(&buf, NULL, start, len, 0, PyBUF_CONTIG) == -1)
+ return -1;
+ memobj = PyMemoryView_FromBuffer(&buf);
+ if (memobj == NULL)
+ return -1;
+ res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readinto, memobj, NULL);
+ Py_DECREF(memobj);
+ if (res == NULL)
+ return -1;
+ if (res == Py_None) {
+ /* Non-blocking stream would have blocked. Special return code! */
+ Py_DECREF(res);
+ return -2;
+ }
+ n = PyNumber_AsSsize_t(res, PyExc_ValueError);
+ Py_DECREF(res);
+ /* TODO: sanity check (0 <= n <= len) */
+ return n;
+}
+
static PyObject *
_BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t n)
{
- PyObject *buf = self->read_buf;
- Py_ssize_t pos = self->read_pos;
- PyObject *data, *chunks, *sep, *res;
- Py_ssize_t current_size;
+ PyObject *data, *chunks, *sep, *res = NULL;
+ Py_ssize_t current_size, remaining, written;
+ unsigned char *out;
/* Special case for when the number of bytes to read is unspecified. */
if (n == -1) {
@@ -425,23 +507,21 @@
if (chunks == NULL)
return NULL;
- Py_INCREF(buf);
- if (_BufferedReader_reset_buf(self) < 0)
- return NULL;
-
- /* Strip the consumed bytes */
- current_size = Py_SIZE(buf) - pos;
+ /* First copy what we have in the current buffer. */
+ current_size = AVAILABLE_BYTES(self);
data = NULL;
if (current_size) {
- data = PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, current_size);
+ data = PyBytes_FromStringAndSize(
+ self->read_buf + self->read_pos, current_size);
if (data == NULL) {
- Py_DECREF(buf);
Py_DECREF(chunks);
return NULL;
}
- Py_DECREF(buf);
}
-
+ if (_BufferedReader_reset_buf(self) == -1) {
+ Py_DECREF(chunks);
+ return NULL;
+ }
while (1) {
if (data) {
if (PyList_Append(chunks, data) < 0) {
@@ -453,21 +533,18 @@
}
/* Read until EOF or until read() would block. */
- data = PyObject_CallMethod(self->raw, "read", NULL);
-
+ data = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_read, NULL);
if (data == NULL) {
Py_DECREF(chunks);
return NULL;
}
-
if (data != Py_None && !PyBytes_Check(data)) {
Py_DECREF(data);
Py_DECREF(chunks);
PyErr_SetString(PyExc_TypeError, "read() should return bytes");
return NULL;
}
-
- if (data == Py_None || Py_SIZE(data) == 0) {
+ if (data == Py_None || PyBytes_GET_SIZE(data) == 0) {
if (current_size == 0) {
Py_DECREF(chunks);
return data;
@@ -475,7 +552,6 @@
else {
Py_DECREF(data);
sep = PyBytes_FromStringAndSize(NULL, 0);
-
if (sep == NULL) {
Py_DECREF(chunks);
return NULL;
@@ -483,116 +559,106 @@
res =_PyBytes_Join(sep, chunks);
Py_DECREF(sep);
Py_DECREF(chunks);
-
return res;
}
}
-
- current_size += Py_SIZE(data);
+ current_size += PyBytes_GET_SIZE(data);
}
}
/* The number of bytes to read is specified, return at most n bytes. */
-
- current_size = Py_SIZE(buf) - pos; /* Length of the available buffered data. */
+ current_size = AVAILABLE_BYTES(self);
if (n <= current_size) {
/* Fast path: the data to read is fully buffered. */
+ res = PyBytes_FromStringAndSize(self->read_buf + self->read_pos, n);
+ if (res == NULL)
+ goto error;
self->read_pos += n;
- return PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, n);
+ return res;
}
/* Slow path: read from the stream until enough bytes are read,
* or until an EOF occurs or until read() would block.
*/
- chunks = PyList_New(0);
- if (chunks == NULL)
- return NULL;
-
- data = NULL;
-
- if (current_size) {
- data = PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, current_size);
- self->read_pos += current_size;
+ res = PyBytes_FromStringAndSize(NULL, n);
+ if (res == NULL)
+ goto error;
+ out = PyBytes_AS_STRING(res);
+ remaining = n;
+ written = 0;
+ if (current_size > 0) {
+ memcpy(out, self->read_buf + self->read_pos, current_size);
+ remaining -= current_size;
+ written += current_size;
}
-
- while (1) {
- Py_ssize_t wanted;
-
- if (data) {
- if (PyList_Append(chunks, data) < 0) {
- Py_DECREF(data);
- Py_DECREF(chunks);
- return NULL;
- }
- Py_DECREF(data);
- }
-
- if (current_size >= n)
+ if (_BufferedReader_reset_buf(self) == -1)
+ goto error;
+ while (remaining > 0) {
+ /* We want to read a whole block at the end into read_buf.
+ If we had readv() we could do this in one pass. */
+ Py_ssize_t r = MINUS_LAST_BLOCK(self, remaining);
+ if (r == 0)
break;
-
- wanted = n;
- if (wanted < self->buffer_size)
- wanted = self->buffer_size;
-
- data = PyObject_CallMethod(self->raw, "read", "n", wanted);
-
- if (data != Py_None && !PyBytes_Check(data)) {
- Py_DECREF(data);
- Py_DECREF(chunks);
- PyErr_SetString(PyExc_TypeError, "read() should return bytes");
- return NULL;
- }
-
- if (data == Py_None || Py_SIZE(data) == 0) {
+ r = _BufferedReader_raw_read(self, out + written, r);
+ if (r == -1)
+ goto error;
+ if (r == 0 || r == -2) {
/* EOF occurred or read() would block. */
- if (current_size == 0) {
- Py_DECREF(chunks);
- if (_BufferedReader_reset_buf(self) < 0) {
- Py_DECREF(data);
- return NULL;
- }
- return data;
- }
- else {
- Py_DECREF(data);
- break;
+ if (r == 0 || written > 0) {
+ if (_PyBytes_Resize(&res, written))
+ goto error;
+ return res;
}
+ Py_DECREF(res);
+ Py_INCREF(Py_None);
+ return Py_None;
}
- current_size += Py_SIZE(data);
+ remaining -= r;
+ written += r;
}
-
- sep = PyBytes_FromStringAndSize(NULL, 0);
- if (sep == NULL) {
- Py_DECREF(chunks);
- return NULL;
- }
- res =_PyBytes_Join(sep, chunks);
- Py_DECREF(sep);
- Py_DECREF(chunks);
-
- if (Py_SIZE(res) > n) {
- /* Save the extra data in the buffer. */
- self->read_pos = 0;
- buf = self->read_buf;
- self->read_buf = PyBytes_FromStringAndSize(PyBytes_AS_STRING(res) + n, Py_SIZE(res) - n);
- Py_DECREF(buf);
- if (self->read_buf == NULL) {
+ assert(remaining <= self->buffer_size);
+ self->read_pos = 0;
+ self->read_end = 0;
+ while (self->read_end < self->buffer_size) {
+ Py_ssize_t r = _BufferedReader_raw_read(self,
+ self->read_buf + self->read_end,
+ self->buffer_size - self->read_end);
+ if (r == -1)
+ goto error;
+ if (r == 0 || r == -2) {
+ /* EOF occurred or read() would block. */
+ if (r == 0 || written > 0) {
+ if (_PyBytes_Resize(&res, written))
+ goto error;
+ return res;
+ }
Py_DECREF(res);
- return NULL;
+ Py_INCREF(Py_None);
+ return Py_None;
}
-
- /* Truncate the result to the desired length */
- buf = PyBytes_FromStringAndSize(PyBytes_AS_STRING(res), n);
- if (buf == NULL) {
- Py_DECREF(res);
- return NULL;
+ if (remaining > r) {
+ memcpy(out + written, self->read_buf + self->read_end, r);
+ written += r;
+ self->read_pos += r;
+ remaining -= r;
}
-
- Py_DECREF(res);
- res = buf;
+ else if (remaining > 0) {
+ memcpy(out + written, self->read_buf + self->read_end, remaining);
+ written += remaining;
+ self->read_pos += remaining;
+ remaining = 0;
+ }
+ self->read_end += r;
+ if (remaining == 0)
+ /* XXX perhaps we could try again to fill the buffer instead? */
+ break;
}
return res;
+
+error:
+ Py_XDECREF(res);
+ return NULL;
}
static PyObject *
@@ -623,45 +689,31 @@
static PyObject *
_BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t n)
{
- Py_ssize_t have;
-
- if (n > self->buffer_size)
- n = self->buffer_size;
-
- have = Py_SIZE(self->read_buf) - self->read_pos;
-
- if (have < n) {
- Py_ssize_t to_read = self->buffer_size - have;
- PyObject *current = PyObject_CallMethod(self->raw, "read", "n", to_read);
-
- if (current == NULL)
- return NULL;
-
- if (!PyBytes_Check(current)) {
- Py_DECREF(current);
- PyErr_SetString(PyExc_TypeError, "read() should return bytes");
- return NULL;
- }
+ Py_ssize_t have, r;
+ PyObject *res = NULL;
+ unsigned char *out;
- if (Py_SIZE(current) > 0) {
- PyObject *oldbuf = self->read_buf;
- self->read_buf = PyBytes_FromStringAndSize(NULL, have + Py_SIZE(current));
- memcpy(PyBytes_AS_STRING(self->read_buf), PyBytes_AS_STRING(oldbuf) + self->read_pos, have);
- memcpy(PyBytes_AS_STRING(self->read_buf) + have, PyBytes_AS_STRING(current), Py_SIZE(current));
- self->read_pos = 0;
- }
- Py_DECREF(current);
- }
+ have = AVAILABLE_BYTES(self);
+ /* Constraints:
+ 1. we don't want to advance the file position.
+ 2. we don't want to lose block alignment, so we can't shift the buffer
+ to make some place.
+ Therefore, we either return `have` bytes (if > 0), or a full buffer.
+ */
+ if (have > 0) {
+ return PyBytes_FromStringAndSize(self->read_buf + self->read_pos, have);
+ }
+
+ /* Fill the buffer from the raw stream, and copy it to the result. */
+ r = _BufferedReader_raw_read(self, self->read_buf, self->buffer_size);
+ if (r == -1)
+ return NULL;
+ if (r == -2)
+ r = 0;
+ self->read_pos = 0;
+ self->read_end = r;
+ return PyBytes_FromStringAndSize(self->read_buf, r);
- if (self->read_pos == 0) {
- Py_INCREF(self->read_buf);
- return self->read_buf;
- }
- else {
- return PyBytes_FromStringAndSize(
- PyBytes_AS_STRING(self->read_buf) + self->read_pos,
- Py_SIZE(self->read_buf) - self->read_pos);
- }
}
static PyObject *
@@ -695,17 +747,26 @@
return NULL;
}
+ /* TODO: n < 0 should raise an error (?) */
if (n <= 0)
return PyBytes_FromStringAndSize(NULL, 0);
ENTER_BUFFERED_READER(self)
+ /* Return up to n bytes. If at least one byte is buffered, we
+ only return buffered bytes. Otherwise, we do one raw read. */
+
+ /* TODO: this mimicks the io.py implementation but is probably wrong.
+ If we need to read from the raw stream, then we should actually read
+ all `n` bytes asked by the caller (and possibly more, so as to fill
+ our buffer for the next reads). */
+
res = _BufferedReader_peek_unlocked(self, 1);
- if(res == NULL)
+ if (res == NULL)
goto end;
Py_DECREF(res);
- have = Py_SIZE(self->read_buf) - self->read_pos;
+ have = self->read_end - self->read_pos;
if (n > have)
n = have;
@@ -719,25 +780,50 @@
static PyObject *
BufferedReader_seek(BufferedObject *self, PyObject *args)
{
- Py_ssize_t pos;
+ Py_ssize_t current, target, avail;
int whence = 0;
- PyObject *res;
+ PyObject *res = NULL;
CHECK_INITIALIZED(self)
- if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &whence)) {
+ if (!PyArg_ParseTuple(args, "n|i:seek", &target, &whence)) {
return NULL;
}
+ /* TODO: sanity check whence value */
ENTER_BUFFERED_READER(self)
- if (whence == 1) {
- pos -= Py_SIZE(self->read_buf) - self->read_pos;
+ /* TODO: cache the value somewhere rather than always query it*/
+ current = _Buffered_raw_tell(self);
+ if (current == -1)
+ goto end;
+ avail = AVAILABLE_BYTES(self);
+ if (avail > 0 && whence != 2) {
+ /* Check if seeking leaves us inside the current buffer,
+ so as to return quickly if possible.
+ Don't know how to do that when whence == 2, though. */
+ Py_ssize_t offset;
+ if (whence == 0)
+ offset = target - (current - avail);
+ else
+ offset = target;
+ if (offset >= -self->read_pos && offset <= avail) {
+ self->read_pos += offset;
+ res = PyLong_FromSsize_t(current - avail + offset);
+ goto end;
+ }
}
- res = PyObject_CallMethod(self->raw, "seek", "ni", pos, whence);
+ /* Fallback: invoke raw seek() method and clear buffer */
+ if (whence == 1)
+ target -= AVAILABLE_BYTES(self);
+
+ /* TODO: align on block boundary and read buffer if needed? */
+ res = PyObject_CallMethod(self->raw, "seek", "ni", target, whence);
if (res == NULL)
goto end;
+ /* TODO: optimize this for situations where seeking still leaves us
+ inside the current buffer. */
if (_BufferedReader_reset_buf(self) < 0)
Py_CLEAR(res);
@@ -749,24 +835,15 @@
static PyObject *
BufferedReader_tell(BufferedObject *self, PyObject *args)
{
- PyObject *op1, *op2, *res;
+ Py_ssize_t pos;
CHECK_INITIALIZED(self)
- op1 = PyObject_CallMethod(self->raw, "tell", NULL);
-
- if (op1 == NULL)
+ pos = _Buffered_raw_tell(self);
+ if (pos == -1)
return NULL;
-
- op2 = PyLong_FromSsize_t(Py_SIZE(self->read_buf) - self->read_pos);
- if (op2 == NULL) {
- Py_DECREF(op1);
- return NULL;
- }
-
- res = PyNumber_Subtract(op1, op2);
- Py_DECREF(op1);
- Py_DECREF(op2);
- return res;
+ pos -= AVAILABLE_BYTES(self);
+ /* TODO: sanity check (pos >= 0) */
+ return PyLong_FromSsize_t(pos);
}
static PyMethodDef BufferedReader_methods[] = {
@@ -886,7 +963,7 @@
self->raw = raw;
self->buffer_size = buffer_size;
- if (_BufferedIOMixin_init(self) < 0)
+ if (_Buffered_init(self) < 0)
return -1;
if (self->write_buf)
@@ -909,25 +986,6 @@
return 0;
}
-/* Returns the address of the `written` member if a BlockingIOError was
- raised, NULL otherwise. The error is always re-raised. */
-static Py_ssize_t *
-_Buffered_check_blocking_error(void)
-{
- PyObject *t, *v, *tb;
- PyBlockingIOErrorObject *err;
-
- PyErr_Fetch(&t, &v, &tb);
- if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) {
- PyErr_Restore(t, v, tb);
- return NULL;
- }
- err = (PyBlockingIOErrorObject *) v;
- /* TODO: sanity check (err->written >= 0) */
- PyErr_Restore(t, v, tb);
- return &err->written;
-}
-
static Py_ssize_t
_BufferedWriter_raw_write(BufferedObject *self, unsigned char *start, Py_ssize_t len)
{
@@ -1517,12 +1575,20 @@
if (_PyIOBase_checkWritable(raw, NULL) == NULL)
return -1;
+ Py_CLEAR(self->raw);
Py_INCREF(raw);
self->raw = raw;
-
self->buffer_size = buffer_size;
- if( _BufferedReader_reset_buf(self) < 0)
+ if (_Buffered_init(self) < 0)
+ return -1;
+ Py_CLEAR(self->read_buf);
+ self->read_buf = PyMem_Malloc(self->buffer_size);
+ if (self->read_buf == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ if (_BufferedReader_reset_buf(self) < 0)
return -1;
self->read_lock = PyThread_allocate_lock();
@@ -1537,8 +1603,6 @@
PyErr_NoMemory();
return -1;
}
- if (_BufferedIOMixin_init(self) < 0)
- return -1;
if (_BufferedWriter_reset_buf(self) < 0)
return -1;
@@ -1689,15 +1753,17 @@
static PyObject *
BufferedRandom_write(BufferedObject *self, PyObject *args)
{
+ Py_ssize_t readahead;
CHECK_INITIALIZED(self)
- if (Py_SIZE(self->read_buf) > 0) {
+ readahead = AVAILABLE_BYTES(self);
+ if (readahead > 0) {
PyObject *res;
/* Undo readahead */
ENTER_BUFFERED_READER(self)
res = PyObject_CallMethod(self->raw, "seek", "ni",
- self->read_pos - Py_SIZE(self->read_buf), 1);
+ -readahead, 1);
Py_XDECREF(res);
if (res != NULL) {
if (_BufferedReader_reset_buf(self) < 0)
Modified: sandbox/trunk/io-c/_iobase.c
==============================================================================
--- sandbox/trunk/io-c/_iobase.c (original)
+++ sandbox/trunk/io-c/_iobase.c Wed Dec 31 00:38:13 2008
@@ -327,9 +327,7 @@
if (has_peek) {
PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1);
-
- assert (PyBytes_Check(readahead));
-
+ assert(PyBytes_Check(readahead));
if (readahead == NULL)
goto fail;
if (PyBytes_GET_SIZE(readahead) > 0) {
@@ -339,37 +337,27 @@
do {
if (n >= PyBytes_GET_SIZE(readahead) || n >= limit)
break;
- if (buf[n] == '\n')
- {
- n++;
+ if (buf[n++] == '\n')
break;
- }
- n++;
} while (1);
}
else {
do {
if (n >= PyBytes_GET_SIZE(readahead))
break;
- if (buf[n] == '\n')
- {
- n++;
+ if (buf[n++] == '\n')
break;
- }
- n++;
} while (1);
}
nreadahead = n;
}
+ Py_DECREF(readahead);
}
b = PyObject_CallMethod(self, "read", "n", nreadahead);
-
if (b == NULL)
goto fail;
-
- assert (PyBytes_Check(b));
-
+ assert(PyBytes_Check(b));
if (Py_SIZE(b) == 0)
break;
Modified: sandbox/trunk/io-c/_iomodule.h
==============================================================================
--- sandbox/trunk/io-c/_iomodule.h (original)
+++ sandbox/trunk/io-c/_iomodule.h Wed Dec 31 00:38:13 2008
@@ -54,6 +54,8 @@
extern PyObject *_PyIO_str_flush;
extern PyObject *_PyIO_str_read;
extern PyObject *_PyIO_str_readable;
+extern PyObject *_PyIO_str_readinto;
extern PyObject *_PyIO_str_seekable;
+extern PyObject *_PyIO_str_tell;
extern PyObject *_PyIO_str_writable;
extern PyObject *_PyIO_str_write;
Modified: sandbox/trunk/io-c/build.py
==============================================================================
--- sandbox/trunk/io-c/build.py (original)
+++ sandbox/trunk/io-c/build.py Wed Dec 31 00:38:13 2008
@@ -30,8 +30,12 @@
p.wait()
print("run the test suite")
- p = subprocess.Popen([sys.executable, "-c",
- "from test.test_io import test_main; test_main()"],
+ #p = subprocess.Popen([sys.executable, "-c",
+ #"from test.test_io import test_main; test_main()"],
+ #env=env)
+
+ # Run our own improved & fixed version of test_io.py.
+ p = subprocess.Popen([sys.executable, "test_io.py"],
env=env)
p.wait()
Modified: sandbox/trunk/io-c/io.c
==============================================================================
--- sandbox/trunk/io-c/io.c (original)
+++ sandbox/trunk/io-c/io.c Wed Dec 31 00:38:13 2008
@@ -10,7 +10,9 @@
PyObject *_PyIO_str_flush;
PyObject *_PyIO_str_read;
PyObject *_PyIO_str_readable;
+PyObject *_PyIO_str_readinto;
PyObject *_PyIO_str_seekable;
+PyObject *_PyIO_str_tell;
PyObject *_PyIO_str_writable;
PyObject *_PyIO_str_write;
@@ -618,8 +620,12 @@
goto fail;
if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable")))
goto fail;
+ if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto")))
+ goto fail;
if (!(_PyIO_str_seekable = PyUnicode_InternFromString("seekable")))
goto fail;
+ if (!(_PyIO_str_tell = PyUnicode_InternFromString("tell")))
+ goto fail;
if (!(_PyIO_str_write = PyUnicode_InternFromString("write")))
goto fail;
if (!(_PyIO_str_writable = PyUnicode_InternFromString("writable")))
Modified: sandbox/trunk/io-c/test_io.py
==============================================================================
--- sandbox/trunk/io-c/test_io.py (original)
+++ sandbox/trunk/io-c/test_io.py Wed Dec 31 00:38:13 2008
@@ -49,6 +49,25 @@
def tell(self):
return 42
+ def readinto(self, buf):
+ max_len = len(buf)
+ try:
+ data = self._read_stack[0]
+ except IndexError:
+ return 0
+ if data is None:
+ del self._read_stack[0]
+ return None
+ n = len(data)
+ if len(data) <= max_len:
+ del self._read_stack[0]
+ buf[:n] = data
+ return n
+ else:
+ buf[:] = data[:max_len]
+ self._read_stack[0] = data[max_len:]
+ return max_len
+
class MockFileIO(io.BytesIO):
@@ -61,6 +80,10 @@
self.read_history.append(None if res is None else len(res))
return res
+ def readinto(self, b):
+ res = io.BytesIO.readinto(self, b)
+ self.read_history.append(res)
+ return res
class MockNonBlockWriterIO(io.RawIOBase):
@@ -415,6 +438,7 @@
for nbytes in buf_read_sizes:
self.assertEquals(bufio.read(nbytes), data[pos:pos+nbytes])
pos += nbytes
+ # this is mildly implementation-dependent
self.assertEquals(rawio.read_history, raw_read_sizes)
def testReadNonBlocking(self):
@@ -1434,11 +1458,14 @@
def test_main():
- support.run_unittest(IOTest, BytesIOTest, StringIOTest,
+ support.run_unittest(
+ IOTest, BytesIOTest, StringIOTest,
BufferedReaderTest, BufferedWriterTest,
BufferedRWPairTest, BufferedRandomTest,
StatefulIncrementalDecoderTest,
- TextIOWrapperTest, MiscIOTest)
+ TextIOWrapperTest, MiscIOTest
+ )
if __name__ == "__main__":
- unittest.main()
+ test_main()
+ #unittest.main()
