[Python-checkins] r68086 - in sandbox/trunk/io-c: _bufferedio.c _iobase.c _iomodule.h build.py io.c test_io.py

antoine.pitrou python-checkins at python.org
Wed Dec 31 00:38:13 CET 2008


Author: antoine.pitrou
Date: Wed Dec 31 00:38:13 2008
New Revision: 68086

Log:
Rewritten BufferedWriter with a static buffer, and misc changes.



Modified:
   sandbox/trunk/io-c/_bufferedio.c
   sandbox/trunk/io-c/_iobase.c
   sandbox/trunk/io-c/_iomodule.h
   sandbox/trunk/io-c/build.py
   sandbox/trunk/io-c/io.c
   sandbox/trunk/io-c/test_io.py

Modified: sandbox/trunk/io-c/_bufferedio.c
==============================================================================
--- sandbox/trunk/io-c/_bufferedio.c	(original)
+++ sandbox/trunk/io-c/_bufferedio.c	Wed Dec 31 00:38:13 2008
@@ -109,8 +109,14 @@
     PyObject *raw;
     int ok;    /* Initialized? */
 
-    PyObject *read_buf;
+    /* A static buffer of size `buffer_size` */
+    unsigned char *read_buf;
+    /* Current reading pos in the buffer. */
     Py_ssize_t read_pos;
+    /* Just after the last buffered byte in the buffer, or -1 if the buffer
+       is invalid.
+       Should most often be equal to buffer_size for "normal" blocking I/O. */
+    Py_ssize_t read_end;
     PyThread_type_lock read_lock;
 
     /* A static buffer of size `buffer_size` */
@@ -122,8 +128,6 @@
     PyThread_type_lock write_lock;
 
     Py_ssize_t buffer_size;
-    /* Still unused, but will help us align reads and writes on block
-       boundaries if desireable */
     Py_ssize_t buffer_mask;
 
     PyObject *dict;
@@ -162,13 +166,28 @@
         return -1; \
     }
 
+#define VALID_READ_BUFFER(self) \
+    (self->read_end != -1)
+
+#define AVAILABLE_BYTES(self) \
+    (VALID_READ_BUFFER(self) ? (self->read_end - self->read_pos) : 0)
+
+#define MINUS_LAST_BLOCK(self, size) \
+    (self->buffer_mask ? \
+        (size & ~self->buffer_mask) : \
+        (self->buffer_size * (size / self->buffer_size)))
+
+
 static void
 BufferedObject_dealloc(BufferedObject *self)
 {
     if (self->weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject *)self);
     Py_CLEAR(self->raw);
-    Py_CLEAR(self->read_buf);
+    if (self->read_buf) {
+        PyMem_Free(self->read_buf);
+        self->write_buf = NULL;
+    }
     if (self->write_buf) {
         PyMem_Free(self->write_buf);
         self->write_buf = NULL;
@@ -331,8 +350,12 @@
     return PyObject_CallMethod(self->raw, "isatty", NULL);
 }
 
+/*
+ * Helpers
+ */
+
 static int
-_BufferedIOMixin_init(BufferedObject *self)
+_Buffered_init(BufferedObject *self)
 {
     Py_ssize_t n;
     if (self->buffer_size <= 0) {
@@ -350,6 +373,40 @@
     return 0;
 }
 
+/* Returns the address of the `written` member if a BlockingIOError was
+   raised, NULL otherwise. The error is always re-raised. */
+static Py_ssize_t *
+_Buffered_check_blocking_error(void)
+{
+    PyObject *t, *v, *tb;
+    PyBlockingIOErrorObject *err;
+
+    PyErr_Fetch(&t, &v, &tb);
+    if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) {
+        PyErr_Restore(t, v, tb);
+        return NULL;
+    }
+    err = (PyBlockingIOErrorObject *) v;
+    /* TODO: sanity check (err->written >= 0) */
+    PyErr_Restore(t, v, tb);
+    return &err->written;
+}
+
+static Py_ssize_t
+_Buffered_raw_tell(BufferedObject *self)
+{
+    PyObject *res;
+    Py_ssize_t n;
+    res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_tell, NULL);
+    if (res == NULL)
+        return -1;
+    n = PyNumber_AsSsize_t(res, PyExc_ValueError);
+    Py_DECREF(res);
+    /* TODO: sanity check (n >= 0) */
+    return n;
+}
+
+
 /*
  * class BufferedReader
  */
@@ -359,16 +416,8 @@
 
 static int _BufferedReader_reset_buf(BufferedObject *self)
 {
-    PyObject *oldbuf = self->read_buf;
-
-    self->read_buf = PyBytes_FromStringAndSize(NULL, 0);
+    self->read_end = -1;
     self->read_pos = 0;
-
-    Py_XDECREF(oldbuf);
-
-    if (self->read_buf == NULL)
-        return -1;
-
     return 0;
 }
 
@@ -392,12 +441,19 @@
     Py_CLEAR(self->raw);
     Py_INCREF(raw);
     self->raw = raw;
-
     self->buffer_size = buffer_size;
-    self->read_buf = NULL;
 
-    if (_BufferedIOMixin_init(self) < 0)
+    if (_Buffered_init(self) < 0)
+        return -1;
+    
+    if (self->read_buf)
+        PyMem_Free(self->read_buf);
+    self->read_buf = PyMem_Malloc(self->buffer_size);
+    if (self->read_buf == NULL) {
+        PyErr_NoMemory();
         return -1;
+    }
+
     if (_BufferedReader_reset_buf(self) < 0)
         return -1;
 
@@ -411,13 +467,39 @@
     return 0;
 }
 
+static Py_ssize_t
+_BufferedReader_raw_read(BufferedObject *self, unsigned char *start, Py_ssize_t len)
+{
+    Py_buffer buf;
+    PyObject *memobj, *res;
+    Py_ssize_t n;
+    /* NOTE: the buffer needn't be released as its object is NULL. */
+    if (PyBuffer_FillInfo(&buf, NULL, start, len, 0, PyBUF_CONTIG) == -1)
+        return -1;
+    memobj = PyMemoryView_FromBuffer(&buf);
+    if (memobj == NULL)
+        return -1;
+    res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readinto, memobj, NULL);
+    Py_DECREF(memobj);
+    if (res == NULL)
+        return -1;
+    if (res == Py_None) {
+        /* Non-blocking stream would have blocked. Special return code! */
+        Py_DECREF(res);
+        return -2;
+    }
+    n = PyNumber_AsSsize_t(res, PyExc_ValueError);
+    Py_DECREF(res);
+    /* TODO: sanity check (0 <= n <= len) */
+    return n;
+}
+
 static PyObject *
 _BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t n)
 {
-    PyObject *buf = self->read_buf;
-    Py_ssize_t pos = self->read_pos;
-    PyObject *data, *chunks, *sep, *res;
-    Py_ssize_t current_size;
+    PyObject *data, *chunks, *sep, *res = NULL;
+    Py_ssize_t current_size, remaining, written;
+    unsigned char *out;
 
     /* Special case for when the number of bytes to read is unspecified. */
     if (n == -1) {
@@ -425,23 +507,21 @@
         if (chunks == NULL)
             return NULL;
 
-                Py_INCREF(buf);
-        if (_BufferedReader_reset_buf(self) < 0)
-            return NULL;
-
-        /* Strip the consumed bytes */
-        current_size = Py_SIZE(buf) - pos;
+        /* First copy what we have in the current buffer. */
+        current_size = AVAILABLE_BYTES(self);
         data = NULL;
         if (current_size) {
-            data = PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, current_size);
+            data = PyBytes_FromStringAndSize(
+                self->read_buf + self->read_pos, current_size);
             if (data == NULL) {
-                                Py_DECREF(buf);
                 Py_DECREF(chunks);
                 return NULL;
             }
-                        Py_DECREF(buf);
         }
-
+        if (_BufferedReader_reset_buf(self) == -1) {
+            Py_DECREF(chunks);
+            return NULL;
+        }
         while (1) {
             if (data) {
                 if (PyList_Append(chunks, data) < 0) {
@@ -453,21 +533,18 @@
             }
 
             /* Read until EOF or until read() would block. */
-            data = PyObject_CallMethod(self->raw, "read", NULL);
-
+            data = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_read, NULL);
             if (data == NULL) {
                 Py_DECREF(chunks);
                 return NULL;
             }
-
             if (data != Py_None && !PyBytes_Check(data)) {
                 Py_DECREF(data);
                 Py_DECREF(chunks);
                 PyErr_SetString(PyExc_TypeError, "read() should return bytes");
                 return NULL;
             }
-
-            if (data == Py_None || Py_SIZE(data) == 0) {
+            if (data == Py_None || PyBytes_GET_SIZE(data) == 0) {
                 if (current_size == 0) {
                     Py_DECREF(chunks);
                     return data;
@@ -475,7 +552,6 @@
                 else {
                     Py_DECREF(data);
                     sep = PyBytes_FromStringAndSize(NULL, 0);
-
                     if (sep == NULL) {
                         Py_DECREF(chunks);
                         return NULL;
@@ -483,116 +559,106 @@
                     res =_PyBytes_Join(sep, chunks);
                     Py_DECREF(sep);
                     Py_DECREF(chunks);
-
                     return res;
                 }
             }
-
-                        current_size += Py_SIZE(data);
+            current_size += PyBytes_GET_SIZE(data);
         }
     }
 
     /* The number of bytes to read is specified, return at most n bytes. */
-
-    current_size = Py_SIZE(buf) - pos; /* Length of the available buffered data. */
+    current_size = AVAILABLE_BYTES(self);
     if (n <= current_size) {
         /* Fast path: the data to read is fully buffered. */
+        res = PyBytes_FromStringAndSize(self->read_buf + self->read_pos, n);
+        if (res == NULL)
+            goto error;
         self->read_pos += n;
-        return PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, n);
+        return res;
     }
 
     /* Slow path: read from the stream until enough bytes are read,
      * or until an EOF occurs or until read() would block.
      */
-    chunks = PyList_New(0);
-    if (chunks == NULL)
-        return NULL;
-
-    data = NULL;
-
-    if (current_size) {
-        data = PyBytes_FromStringAndSize(PyBytes_AS_STRING(buf) + pos, current_size);
-        self->read_pos += current_size;
+    res = PyBytes_FromStringAndSize(NULL, n);
+    if (res == NULL)
+        goto error;
+    out = PyBytes_AS_STRING(res);
+    remaining = n;
+    written = 0;
+    if (current_size > 0) {
+        memcpy(out, self->read_buf + self->read_pos, current_size);
+        remaining -= current_size;
+        written += current_size;
     }
-
-    while (1) {
-        Py_ssize_t wanted;
-
-        if (data) {
-            if (PyList_Append(chunks, data) < 0) {
-                Py_DECREF(data);
-                Py_DECREF(chunks);
-                return NULL;
-            }
-            Py_DECREF(data);
-        }
-
-        if (current_size >= n)
+    if (_BufferedReader_reset_buf(self) == -1)
+        goto error;
+    while (remaining > 0) {
+        /* We want to read a whole block at the end into read_buf.
+           If we had readv() we could do this in one pass. */
+        Py_ssize_t r = MINUS_LAST_BLOCK(self, remaining);
+        if (r == 0)
             break;
-
-        wanted = n;
-        if (wanted < self->buffer_size)
-            wanted = self->buffer_size;
-
-        data = PyObject_CallMethod(self->raw, "read", "n", wanted);
-
-        if (data != Py_None && !PyBytes_Check(data)) {
-            Py_DECREF(data);
-            Py_DECREF(chunks);
-            PyErr_SetString(PyExc_TypeError, "read() should return bytes");
-            return NULL;
-        }
-
-        if (data == Py_None || Py_SIZE(data) == 0) {
+        r = _BufferedReader_raw_read(self, out + written, r);
+        if (r == -1)
+            goto error;
+        if (r == 0 || r == -2) {
             /* EOF occurred or read() would block. */
-            if (current_size == 0) {
-                Py_DECREF(chunks);
-                if (_BufferedReader_reset_buf(self) < 0) {
-                    Py_DECREF(data);
-                    return NULL;
-                }
-                return data;
-            }
-            else {
-                Py_DECREF(data);
-                break;
+            if (r == 0 || written > 0) {
+                if (_PyBytes_Resize(&res, written))
+                    goto error;
+                return res;
             }
+            Py_DECREF(res);
+            Py_INCREF(Py_None);
+            return Py_None;
         }
-        current_size += Py_SIZE(data);
+        remaining -= r;
+        written += r;
     }
-
-    sep = PyBytes_FromStringAndSize(NULL, 0);
-    if (sep == NULL) {
-        Py_DECREF(chunks);
-        return NULL;
-    }
-    res =_PyBytes_Join(sep, chunks);
-    Py_DECREF(sep);
-    Py_DECREF(chunks);
-
-    if (Py_SIZE(res) > n) {
-        /* Save the extra data in the buffer. */
-        self->read_pos = 0;
-        buf = self->read_buf;
-        self->read_buf = PyBytes_FromStringAndSize(PyBytes_AS_STRING(res) + n, Py_SIZE(res) - n);
-        Py_DECREF(buf);
-        if (self->read_buf == NULL) {
+    assert(remaining <= self->buffer_size);
+    self->read_pos = 0;
+    self->read_end = 0;
+    while (self->read_end < self->buffer_size) {
+        Py_ssize_t r = _BufferedReader_raw_read(self, 
+            self->read_buf + self->read_end,
+            self->buffer_size - self->read_end);
+        if (r == -1)
+            goto error;
+        if (r == 0 || r == -2) {
+            /* EOF occurred or read() would block. */
+            if (r == 0 || written > 0) {
+                if (_PyBytes_Resize(&res, written))
+                    goto error;
+                return res;
+            }
             Py_DECREF(res);
-            return NULL;
+            Py_INCREF(Py_None);
+            return Py_None;
         }
-
-        /* Truncate the result to the desired length */
-        buf = PyBytes_FromStringAndSize(PyBytes_AS_STRING(res), n);
-        if (buf == NULL) {
-            Py_DECREF(res);
-            return NULL;
+        if (remaining > r) {
+            memcpy(out + written, self->read_buf + self->read_end, r);
+            written += r;
+            self->read_pos += r;
+            remaining -= r;
         }
-
-        Py_DECREF(res);
-        res = buf;
+        else if (remaining > 0) {
+            memcpy(out + written, self->read_buf + self->read_end, remaining);
+            written += remaining;
+            self->read_pos += remaining;
+            remaining = 0;
+        }
+        self->read_end += r;
+        if (remaining == 0)
+            /* XXX perhaps we could try again to fill the buffer instead? */
+            break;
     }
 
     return res;
+
+error:
+    Py_XDECREF(res);
+    return NULL;
 }
 
 static PyObject *
@@ -623,45 +689,31 @@
 static PyObject *
 _BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t n)
 {
-    Py_ssize_t have;
-
-    if (n > self->buffer_size)
-        n = self->buffer_size;
-
-    have = Py_SIZE(self->read_buf) - self->read_pos;
-
-    if (have < n) {
-        Py_ssize_t to_read = self->buffer_size - have;
-        PyObject *current = PyObject_CallMethod(self->raw, "read", "n", to_read);
-
-        if (current == NULL)
-            return NULL;
-
-        if (!PyBytes_Check(current)) {
-            Py_DECREF(current);
-            PyErr_SetString(PyExc_TypeError, "read() should return bytes");
-            return NULL;
-        }
+    Py_ssize_t have, r;
+    PyObject *res = NULL;
+    unsigned char *out;
 
-        if (Py_SIZE(current) > 0) {
-            PyObject *oldbuf = self->read_buf;
-            self->read_buf = PyBytes_FromStringAndSize(NULL, have + Py_SIZE(current));
-            memcpy(PyBytes_AS_STRING(self->read_buf), PyBytes_AS_STRING(oldbuf) + self->read_pos, have);
-            memcpy(PyBytes_AS_STRING(self->read_buf) + have, PyBytes_AS_STRING(current), Py_SIZE(current));
-            self->read_pos = 0;
-        }
-        Py_DECREF(current);
-    }
+    have = AVAILABLE_BYTES(self);
+    /* Constraints:
+       1. we don't want to advance the file position.
+       2. we don't want to lose block alignment, so we can't shift the buffer
+          to make some place.
+       Therefore, we either return `have` bytes (if > 0), or a full buffer.
+    */
+    if (have > 0) {
+        return PyBytes_FromStringAndSize(self->read_buf + self->read_pos, have);
+    }
+
+    /* Fill the buffer from the raw stream, and copy it to the result. */
+    r = _BufferedReader_raw_read(self, self->read_buf, self->buffer_size);
+    if (r == -1)
+        return NULL;
+    if (r == -2)
+        r = 0;
+    self->read_pos = 0;
+    self->read_end = r;
+    return PyBytes_FromStringAndSize(self->read_buf, r);
 
-    if (self->read_pos == 0) {
-        Py_INCREF(self->read_buf);
-        return self->read_buf;
-    }
-    else {
-        return PyBytes_FromStringAndSize(
-            PyBytes_AS_STRING(self->read_buf) + self->read_pos,
-            Py_SIZE(self->read_buf) - self->read_pos);
-    }
 }
 
 static PyObject *
@@ -695,17 +747,26 @@
         return NULL;
     }
 
+    /* TODO: n < 0 should raise an error (?) */
     if (n <= 0)
         return PyBytes_FromStringAndSize(NULL, 0);
 
     ENTER_BUFFERED_READER(self)
 
+    /* Return up to n bytes.  If at least one byte is buffered, we
+       only return buffered bytes.  Otherwise, we do one raw read. */
+
+    /* TODO: this mimicks the io.py implementation but is probably wrong.
+       If we need to read from the raw stream, then we should actually read
+       all `n` bytes asked by the caller (and possibly more, so as to fill
+       our buffer for the next reads). */
+
     res = _BufferedReader_peek_unlocked(self, 1);
-    if(res == NULL)
+    if (res == NULL)
         goto end;
     Py_DECREF(res);
 
-    have = Py_SIZE(self->read_buf) - self->read_pos;
+    have = self->read_end - self->read_pos;
     if (n > have)
         n = have;
 
@@ -719,25 +780,50 @@
 static PyObject *
 BufferedReader_seek(BufferedObject *self, PyObject *args)
 {
-    Py_ssize_t pos;
+    Py_ssize_t current, target, avail;
     int whence = 0;
-    PyObject *res;
+    PyObject *res = NULL;
 
     CHECK_INITIALIZED(self)
-    if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &whence)) {
+    if (!PyArg_ParseTuple(args, "n|i:seek", &target, &whence)) {
         return NULL;
     }
+    /* TODO: sanity check whence value */
 
     ENTER_BUFFERED_READER(self)
 
-    if (whence == 1) {
-        pos -= Py_SIZE(self->read_buf) - self->read_pos;
+    /* TODO: cache the value somewhere rather than always query it*/
+    current = _Buffered_raw_tell(self);
+    if (current == -1)
+        goto end;
+    avail = AVAILABLE_BYTES(self);
+    if (avail > 0 && whence != 2) {
+        /* Check if seeking leaves us inside the current buffer,
+           so as to return quickly if possible.
+           Don't know how to do that when whence == 2, though. */
+        Py_ssize_t offset;
+        if (whence == 0)
+            offset = target - (current - avail);
+        else
+            offset = target;
+        if (offset >= -self->read_pos && offset <= avail) {
+            self->read_pos += offset;
+            res = PyLong_FromSsize_t(current - avail + offset);
+            goto end;
+        }
     }
 
-    res = PyObject_CallMethod(self->raw, "seek", "ni", pos, whence);
+    /* Fallback: invoke raw seek() method and clear buffer */
+    if (whence == 1)
+        target -= AVAILABLE_BYTES(self);
+
+    /* TODO: align on block boundary and read buffer if needed? */
+    res = PyObject_CallMethod(self->raw, "seek", "ni", target, whence);
     if (res == NULL)
         goto end;
 
+    /* TODO: optimize this for situations where seeking still leaves us
+       inside the current buffer. */
     if (_BufferedReader_reset_buf(self) < 0)
         Py_CLEAR(res);
 
@@ -749,24 +835,15 @@
 static PyObject *
 BufferedReader_tell(BufferedObject *self, PyObject *args)
 {
-    PyObject *op1, *op2, *res;
+    Py_ssize_t pos;
 
     CHECK_INITIALIZED(self)
-    op1 = PyObject_CallMethod(self->raw, "tell", NULL);
-
-    if (op1 == NULL)
+    pos = _Buffered_raw_tell(self);
+    if (pos == -1)
         return NULL;
-
-    op2 = PyLong_FromSsize_t(Py_SIZE(self->read_buf) - self->read_pos);
-    if (op2 == NULL) {
-        Py_DECREF(op1);
-        return NULL;
-    }
-
-    res = PyNumber_Subtract(op1, op2);
-    Py_DECREF(op1);
-    Py_DECREF(op2);
-    return res;
+    pos -= AVAILABLE_BYTES(self);
+    /* TODO: sanity check (pos >= 0) */
+    return PyLong_FromSsize_t(pos);
 }
 
 static PyMethodDef BufferedReader_methods[] = {
@@ -886,7 +963,7 @@
     self->raw = raw;
 
     self->buffer_size = buffer_size;
-    if (_BufferedIOMixin_init(self) < 0)
+    if (_Buffered_init(self) < 0)
         return -1;
 
     if (self->write_buf)
@@ -909,25 +986,6 @@
     return 0;
 }
 
-/* Returns the address of the `written` member if a BlockingIOError was
-   raised, NULL otherwise. The error is always re-raised. */
-static Py_ssize_t *
-_Buffered_check_blocking_error(void)
-{
-    PyObject *t, *v, *tb;
-    PyBlockingIOErrorObject *err;
-
-    PyErr_Fetch(&t, &v, &tb);
-    if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) {
-        PyErr_Restore(t, v, tb);
-        return NULL;
-    }
-    err = (PyBlockingIOErrorObject *) v;
-    /* TODO: sanity check (err->written >= 0) */
-    PyErr_Restore(t, v, tb);
-    return &err->written;
-}
-
 static Py_ssize_t
 _BufferedWriter_raw_write(BufferedObject *self, unsigned char *start, Py_ssize_t len)
 {
@@ -1517,12 +1575,20 @@
     if (_PyIOBase_checkWritable(raw, NULL) == NULL)
         return -1;
 
+    Py_CLEAR(self->raw);
     Py_INCREF(raw);
     self->raw = raw;
-
     self->buffer_size = buffer_size;
 
-    if( _BufferedReader_reset_buf(self) < 0)
+    if (_Buffered_init(self) < 0)
+        return -1;
+    Py_CLEAR(self->read_buf);
+    self->read_buf = PyMem_Malloc(self->buffer_size);
+    if (self->read_buf == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    if (_BufferedReader_reset_buf(self) < 0)
         return -1;
 
     self->read_lock = PyThread_allocate_lock();
@@ -1537,8 +1603,6 @@
         PyErr_NoMemory();
         return -1;
     }
-    if (_BufferedIOMixin_init(self) < 0)
-        return -1;
     if (_BufferedWriter_reset_buf(self) < 0)
         return -1;
 
@@ -1689,15 +1753,17 @@
 static PyObject *
 BufferedRandom_write(BufferedObject *self, PyObject *args)
 {
+    Py_ssize_t readahead;
     CHECK_INITIALIZED(self)
-    if (Py_SIZE(self->read_buf) > 0) {
+    readahead = AVAILABLE_BYTES(self);
+    if (readahead > 0) {
         PyObject *res;
         /* Undo readahead */
 
         ENTER_BUFFERED_READER(self)
     
         res = PyObject_CallMethod(self->raw, "seek", "ni",
-                                  self->read_pos - Py_SIZE(self->read_buf), 1);
+                                  -readahead, 1);
         Py_XDECREF(res);
         if (res != NULL) {
             if (_BufferedReader_reset_buf(self) < 0)

Modified: sandbox/trunk/io-c/_iobase.c
==============================================================================
--- sandbox/trunk/io-c/_iobase.c	(original)
+++ sandbox/trunk/io-c/_iobase.c	Wed Dec 31 00:38:13 2008
@@ -327,9 +327,7 @@
 
         if (has_peek) {
             PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1);
-
-            assert (PyBytes_Check(readahead));
-
+            assert(PyBytes_Check(readahead));
             if (readahead == NULL)
                 goto fail;
             if (PyBytes_GET_SIZE(readahead) > 0) {
@@ -339,37 +337,27 @@
                     do {
                         if (n >= PyBytes_GET_SIZE(readahead) || n >= limit)
                             break;
-                        if (buf[n] == '\n')
-                        {
-                            n++;
+                        if (buf[n++] == '\n')
                             break;
-                        }
-                        n++;
                     } while (1);
                 }
                 else {
                     do {
                         if (n >= PyBytes_GET_SIZE(readahead))
                             break;
-                        if (buf[n] == '\n')
-                        {
-                            n++;
+                        if (buf[n++] == '\n')
                             break;
-                        }
-                        n++;
                     } while (1);
                 }
                 nreadahead = n;
             }
+            Py_DECREF(readahead);
         }
 
         b = PyObject_CallMethod(self, "read", "n", nreadahead);
-
         if (b == NULL)
             goto fail;
-
-        assert (PyBytes_Check(b));
-
+        assert(PyBytes_Check(b));
         if (Py_SIZE(b) == 0)
             break;
 

Modified: sandbox/trunk/io-c/_iomodule.h
==============================================================================
--- sandbox/trunk/io-c/_iomodule.h	(original)
+++ sandbox/trunk/io-c/_iomodule.h	Wed Dec 31 00:38:13 2008
@@ -54,6 +54,8 @@
 extern PyObject *_PyIO_str_flush;
 extern PyObject *_PyIO_str_read;
 extern PyObject *_PyIO_str_readable;
+extern PyObject *_PyIO_str_readinto;
 extern PyObject *_PyIO_str_seekable;
+extern PyObject *_PyIO_str_tell;
 extern PyObject *_PyIO_str_writable;
 extern PyObject *_PyIO_str_write;

Modified: sandbox/trunk/io-c/build.py
==============================================================================
--- sandbox/trunk/io-c/build.py	(original)
+++ sandbox/trunk/io-c/build.py	Wed Dec 31 00:38:13 2008
@@ -30,8 +30,12 @@
     p.wait()
 
     print("run the test suite")
-    p = subprocess.Popen([sys.executable, "-c",
-                          "from test.test_io import test_main; test_main()"],
+    #p = subprocess.Popen([sys.executable, "-c",
+                          #"from test.test_io import test_main; test_main()"],
+                         #env=env)
+
+    # Run our own improved & fixed version of test_io.py.
+    p = subprocess.Popen([sys.executable, "test_io.py"],
                          env=env)
     p.wait()
 

Modified: sandbox/trunk/io-c/io.c
==============================================================================
--- sandbox/trunk/io-c/io.c	(original)
+++ sandbox/trunk/io-c/io.c	Wed Dec 31 00:38:13 2008
@@ -10,7 +10,9 @@
 PyObject *_PyIO_str_flush;
 PyObject *_PyIO_str_read;
 PyObject *_PyIO_str_readable;
+PyObject *_PyIO_str_readinto;
 PyObject *_PyIO_str_seekable;
+PyObject *_PyIO_str_tell;
 PyObject *_PyIO_str_writable;
 PyObject *_PyIO_str_write;
 
@@ -618,8 +620,12 @@
         goto fail;
     if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable")))
         goto fail;
+    if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto")))
+        goto fail;
     if (!(_PyIO_str_seekable = PyUnicode_InternFromString("seekable")))
         goto fail;
+    if (!(_PyIO_str_tell = PyUnicode_InternFromString("tell")))
+        goto fail;
     if (!(_PyIO_str_write = PyUnicode_InternFromString("write")))
         goto fail;
     if (!(_PyIO_str_writable = PyUnicode_InternFromString("writable")))

Modified: sandbox/trunk/io-c/test_io.py
==============================================================================
--- sandbox/trunk/io-c/test_io.py	(original)
+++ sandbox/trunk/io-c/test_io.py	Wed Dec 31 00:38:13 2008
@@ -49,6 +49,25 @@
     def tell(self):
         return 42
 
+    def readinto(self, buf):
+        max_len = len(buf)
+        try:
+            data = self._read_stack[0]
+        except IndexError:
+            return 0
+        if data is None:
+            del self._read_stack[0]
+            return None
+        n = len(data)
+        if len(data) <= max_len:
+            del self._read_stack[0]
+            buf[:n] = data
+            return n
+        else:
+            buf[:] = data[:max_len]
+            self._read_stack[0] = data[max_len:]
+            return max_len
+
 
 class MockFileIO(io.BytesIO):
 
@@ -61,6 +80,10 @@
         self.read_history.append(None if res is None else len(res))
         return res
 
+    def readinto(self, b):
+        res = io.BytesIO.readinto(self, b)
+        self.read_history.append(res)
+        return res
 
 class MockNonBlockWriterIO(io.RawIOBase):
 
@@ -415,6 +438,7 @@
             for nbytes in buf_read_sizes:
                 self.assertEquals(bufio.read(nbytes), data[pos:pos+nbytes])
                 pos += nbytes
+            # this is mildly implementation-dependent
             self.assertEquals(rawio.read_history, raw_read_sizes)
 
     def testReadNonBlocking(self):
@@ -1434,11 +1458,14 @@
 
 
 def test_main():
-    support.run_unittest(IOTest, BytesIOTest, StringIOTest,
+    support.run_unittest(
+        IOTest, BytesIOTest, StringIOTest,
                               BufferedReaderTest, BufferedWriterTest,
                               BufferedRWPairTest, BufferedRandomTest,
                               StatefulIncrementalDecoderTest,
-                              TextIOWrapperTest, MiscIOTest)
+                              TextIOWrapperTest, MiscIOTest
+                              )
 
 if __name__ == "__main__":
-    unittest.main()
+    test_main()
+    #unittest.main()


More information about the Python-checkins mailing list