[Python-checkins] r43642 - python/trunk/Doc/lib/libcsv.tex
walter.doerwald
python-checkins at python.org
Tue Apr 4 19:32:49 CEST 2006
Author: walter.doerwald
Date: Tue Apr 4 19:32:49 2006
New Revision: 43642
Modified:
python/trunk/Doc/lib/libcsv.tex
Log:
Change the example classes UnicodeReader and UnicodeWriter so
that they work with all encodings. For UnicodeReader the real
input stream is wrapped in a line iterator that reencodes the
input to UTF-8. For UnicodeWriter the UTF-8 encoded output is
written to a queue for where it is reencoded to the target
encoding and written to the real output stream.
Modified: python/trunk/Doc/lib/libcsv.tex
==============================================================================
--- python/trunk/Doc/lib/libcsv.tex (original)
+++ python/trunk/Doc/lib/libcsv.tex Tue Apr 4 19:32:49 2006
@@ -456,44 +456,68 @@
yield line.encode('utf-8')
\end{verbatim}
-The classes below work just like the \class{csv.reader} and
-\class{csv.writer} classes, but they add an \var{encoding} parameter
-to allow for encoded files:
+For all other encodings the following \class{UnicodeReader} and
+\class{UnicodeWriter} classes can be used. They take an additional
+\var{encoding} parameter in their constructor and make sure that the data
+passes the real reader or writer encoded as UTF-8:
\begin{verbatim}
-import csv
+import csv, codecs, cStringIO
-class UnicodeReader:
+class UTF8Recoder:
+ """
+ Iterator that reads an encoded stream and reencodes the input to UTF-8
+ """
+ def __init__(self, f, encoding):
+ self.reader = codecs.getreader(encoding)(f)
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return self.reader.next().encode("utf-8")
+class UnicodeReader:
"""
A CSV reader which will iterate over lines in the CSV file "f",
which is encoded in the given encoding.
"""
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+ f = UTF8Recoder(f, encoding)
self.reader = csv.reader(f, dialect=dialect, **kwds)
- self.encoding = encoding
def next(self):
row = self.reader.next()
- return [unicode(s, self.encoding) for s in row]
+ return [unicode(s, "utf-8") for s in row]
def __iter__(self):
return self
class UnicodeWriter:
-
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
- self.writer = csv.writer(f, dialect=dialect, **kwds)
- self.encoding = encoding
+ # Redirect output to a queue
+ self.queue = cStringIO.StringIO()
+ self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
+ self.stream = f
+ self.encoder = codecs.getincrementalencoder(encoding)()
def writerow(self, row):
- self.writer.writerow([s.encode(self.encoding) for s in row])
+ self.writer.writerow([s.encode("utf-8") for s in row])
+ # Fetch UTF-8 output from the queue ...
+ data = self.queue.getvalue()
+ data = data.decode("utf-8")
+ # ... and reencode it into the target encoding
+ data = self.encoder.encode(data)
+ # write to the target stream
+ self.stream.write(data)
+ # empty queue
+ self.queue.truncate(0)
def writerows(self, rows):
for row in rows:
More information about the Python-checkins
mailing list