[Python-checkins] r68625 - sandbox/trunk/io-c/_textio.c

[antoine.pitrou]

Author: antoine.pitrou
Date: Fri Jan 16 01:28:23 2009
New Revision: 68625

Log:
Text I/O: speed up small writes by buffering them instead of 
sending them immediately.



Modified:
   sandbox/trunk/io-c/_textio.c

Modified: sandbox/trunk/io-c/_textio.c
==============================================================================
--- sandbox/trunk/io-c/_textio.c	(original)
+++ sandbox/trunk/io-c/_textio.c	Fri Jan 16 01:28:23 2009
@@ -473,8 +473,20 @@
     int seekable:1;
     int telling:1;
 
+    /* Reads and writes are internally buffered in order to speed things up.
+       However, any read will first flush the write buffer if itsn't empty.
+    
+       Please also note that text to be written is first encoded before being
+       buffered. This is necessary so that encoding errors are immediately
+       reported to the caller, but it unfortunately means that the
+       IncrementalEncoder (whose encode() method is always written in Python)
+       becomes a bottleneck for small writes.
+    */
     PyObject *decoded_chars;       /* buffer for text returned from decoder */
     Py_ssize_t decoded_chars_used; /* offset into _decoded_chars for read() */
+    PyObject *pending_bytes;       /* list of bytes objects waiting to be
+                                      written, or NULL */
+    Py_ssize_t pending_bytes_count;
     PyObject *snapshot;
     /* snapshot is either None, or a tuple (dec_flags, next_input) where
      * dec_flags is the second (integer) item of the decoder state and
@@ -521,6 +533,11 @@
     Py_CLEAR(self->encoder);
     Py_CLEAR(self->decoder);
     Py_CLEAR(self->readnl);
+    Py_CLEAR(self->decoded_chars);
+    Py_CLEAR(self->pending_bytes);
+    Py_CLEAR(self->snapshot);
+    self->decoded_chars_used = 0;
+    self->pending_bytes_count = 0;
 
     if (encoding == NULL) {
         /* Try os.device_encoding(fileno) */
@@ -722,6 +739,29 @@
     return NULL;
 }
 
+/* Flush the internal write buffer. This doesn't explicitly flush the 
+   underlying buffered object, though. */
+static int
+_TextIOWrapper_writeflush(PyTextIOWrapperObject *self)
+{
+    PyObject *b, *ret;
+
+    if (self->pending_bytes == NULL)
+        return 0;
+    b = _PyBytes_Join(PyBytes_FromStringAndSize(NULL, 0), self->pending_bytes);
+    if (b == NULL)
+        return -1;
+    ret = PyObject_CallMethodObjArgs(self->buffer,
+                                     _PyIO_str_write, b, NULL);
+    Py_DECREF(b);
+    if (ret == NULL)
+        return -1;
+    Py_DECREF(ret);
+    Py_CLEAR(self->pending_bytes);
+    self->pending_bytes_count = 0;
+    return 0;
+}
+
 static PyObject *
 TextIOWrapper_write(PyTextIOWrapperObject *self, PyObject *args)
 {
@@ -770,12 +810,25 @@
     if (b == NULL)
         return NULL;
 
-    ret = PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_write, b, NULL);
-    Py_DECREF(b);
-    if (ret == NULL)
+    if (self->pending_bytes == NULL) {
+        self->pending_bytes = PyList_New(0);
+        if (self->pending_bytes == NULL) {
+            Py_DECREF(b);
+            return NULL;
+        }
+        self->pending_bytes_count = 0;
+    }
+    if (PyList_Append(self->pending_bytes, b) < 0) {
+        Py_DECREF(b);
         return NULL;
-    Py_DECREF(ret);
-
+    }
+    self->pending_bytes_count += PyBytes_GET_SIZE(b);
+    Py_DECREF(b);
+    if (self->pending_bytes_count > self->chunk_size || needflush) {
+        if (_TextIOWrapper_writeflush(self) < 0)
+            return NULL;
+    }
+    
     if (needflush) {
         ret = PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_flush, NULL);
         if (ret == NULL)
@@ -968,6 +1021,9 @@
 
     CHECK_CLOSED(self);
 
+    if (_TextIOWrapper_writeflush(self) < 0)
+        return NULL;
+
     if (n < 0) {
         /* Read everything */
         PyObject *bytes = PyObject_CallMethod(self->buffer, "read", NULL);
@@ -1160,6 +1216,9 @@
 
     CHECK_CLOSED(self);
 
+    if (_TextIOWrapper_writeflush(self) < 0)
+        return NULL;
+
     chunked = 0;
 
     while (1) {
@@ -1594,6 +1653,8 @@
         goto fail;
     }
 
+    if (_TextIOWrapper_writeflush(self) < 0)
+        return NULL;
     res = PyObject_CallMethod((PyObject *)self, "flush", NULL);
     if (res == NULL)
         goto fail;
@@ -1783,6 +1844,8 @@
     CHECK_INITIALIZED(self);
     CHECK_CLOSED(self);
     self->telling = self->seekable;
+    if (_TextIOWrapper_writeflush(self) < 0)
+        return NULL;
     return PyObject_CallMethod(self->buffer, "flush", NULL);
 }