[pypy-svn] r30945 - in pypy/dist/pypy/translator/cli: src test
antocuni at codespeak.net
antocuni at codespeak.net
Thu Aug 3 14:55:18 CEST 2006
Author: antocuni
Date: Thu Aug 3 14:55:11 2006
New Revision: 30945
Modified:
pypy/dist/pypy/translator/cli/src/ll_os.cs
pypy/dist/pypy/translator/cli/src/pypylib.cs
pypy/dist/pypy/translator/cli/test/test_builtin.py
Log:
This should fix the support for binary/text files both on unix and
windows. There were many problems:
- the .NET framework seems not to have any support for CRLF
terminated text files, so we need to handle them explicitly;
- we can't use StreamReader and StreamWriter because the are
designed to handle unicode strings: the default encoding is UTF8,
which breaks things when writing chars > 0x7F. The ASCII encoding
doesn't handle chars > 0x7F too, so we can't use it.
The solution is to directly read bytes and casting them to
chars. Moreover, we need to pay attention on Windows when opening
files in text mode, because we need the CRLF conversion. We will
probably need a similar thing for MAC < OS9 in the future.
Modified: pypy/dist/pypy/translator/cli/src/ll_os.cs
==============================================================================
--- pypy/dist/pypy/translator/cli/src/ll_os.cs (original)
+++ pypy/dist/pypy/translator/cli/src/ll_os.cs Thu Aug 3 14:55:11 2006
@@ -13,6 +13,7 @@
string Read(int count);
}
+ // this class is used only for stdin/stdout/stderr
class TextFile: IFile
{
private FileStream stream;
@@ -46,41 +47,45 @@
}
}
- class CRLFFile: IFile
+ class CRLFTextFile: IFile
{
private FileStream stream;
- private TextWriter writer;
- private TextReader reader;
- public CRLFFile(FileStream stream, TextReader reader, TextWriter writer)
+ public CRLFTextFile(FileStream stream)
{
this.stream = stream;
- this.writer = writer;
- this.reader = reader;
}
-
+
public FileStream GetStream()
{
return stream;
}
-
+
public void Write(string buffer)
{
- Debug.Assert(writer != null); // XXX: raise OSError?
- writer.Write(buffer);
- writer.Flush();
+ foreach(char ch in buffer) {
+ if (ch == '\n')
+ stream.WriteByte((byte)'\r');
+ stream.WriteByte((byte)ch);
+ }
+ stream.Flush(); // XXX: should we flush every time or not?
}
-
+
public string Read(int count)
{
- Debug.Assert(reader != null); // XXX: raise OSError?
System.Text.StringBuilder builder = new System.Text.StringBuilder(count);
+ bool pending_CR = false;
while (count-- > 0) {
- int ch = reader.Read();
+ int ch = stream.ReadByte();
if (ch == -1)
break;
- if (ch == '\r' && reader.Peek() == '\n')
- ch = reader.Read();
- builder.Append((char)ch);
+ else if (ch == '\r')
+ pending_CR = true;
+ else {
+ if (pending_CR && ch != '\n')
+ builder.Append('\r');
+ builder.Append((char)ch);
+ pending_CR = false;
+ }
}
return builder.ToString();
}
@@ -136,6 +141,7 @@
static ll_os()
{
FileDescriptors = new Dictionary<int, IFile>();
+ // XXX: what about CRLF conversion for stdin, stdout and stderr?
FileDescriptors[0] = new TextFile(null, Console.In, null);
FileDescriptors[1] = new TextFile(null, null, Console.Out);
FileDescriptors[2] = new TextFile(null, null, Console.Error);
@@ -175,30 +181,13 @@
FileStream stream = new FileStream(name, f_mode, f_access);
IFile f;
- if ((flags & O_BINARY) != 0)
+ // - on Unix there is no difference between text and binary modes
+ // - on Windows text mode means that we should convert '\n' from and to '\r\n'
+ // - on Mac < OS9 text mode means that we should convert '\n' from and to '\r' -- XXX: TODO!
+ if ((flags & O_BINARY) == 0 && System.Environment.NewLine == "\r\n")
+ f = new CRLFTextFile(stream);
+ else
f = new BinaryFile(stream);
- else {
- StreamWriter writer = null;
- StreamReader reader = null;
- if (f_access == FileAccess.Read || f_access == FileAccess.ReadWrite)
- reader = new StreamReader(stream);
- if (f_access == FileAccess.Write || f_access == FileAccess.ReadWrite)
- {
- Console.Error.WriteLine("opening {0} for writing", name);
- writer = new StreamWriter(stream);
- }
-
- if (System.Environment.NewLine == "\r\n")
- {
- Console.Error.WriteLine("opening {0} for reading (with CRLF conversion)", name);
- f = new CRLFFile(stream, reader, writer);
- }
- else
- {
- Console.Error.WriteLine("opening {0} for reading (without CRLF conversion)", name);
- f = new TextFile(stream, reader, writer);
- }
- }
fdcount++;
FileDescriptors[fdcount] = f;
@@ -214,21 +203,22 @@
public static int ll_os_write(int fd, string buffer)
{
- PrintString("ll_os_write", buffer);
getfd(fd).Write(buffer);
return buffer.Length;
}
+ /*
private static void PrintString(string source, string s)
{
- Console.WriteLine(source);
- Console.WriteLine(s);
- Console.WriteLine("Length: {0}", s.Length);
+ Console.Error.WriteLine(source);
+ Console.Error.WriteLine(s);
+ Console.Error.WriteLine("Length: {0}", s.Length);
for (int i=0; i<s.Length; i++)
- Console.Write("{0} ", (int)s[i]);
- Console.WriteLine();
- Console.WriteLine();
+ Console.Error.Write("{0} ", (int)s[i]);
+ Console.Error.WriteLine();
+ Console.Error.WriteLine();
}
+ */
public static string ll_os_read(int fd, long count)
{
Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs
==============================================================================
--- pypy/dist/pypy/translator/cli/src/pypylib.cs (original)
+++ pypy/dist/pypy/translator/cli/src/pypylib.cs Thu Aug 3 14:55:11 2006
@@ -18,8 +18,12 @@
public static string ToPython(string x) {
if (x == null)
return "None";
- else
- return string.Format("'''{0}'''", x.Replace("\\", "\\\\"));
+ else {
+ string res = "";
+ foreach(char ch in x)
+ res+= string.Format("\\x{0:X2}", (int)ch);
+ return string.Format("'{0}'", res);
+ }
}
public static string ToPython(object x) {
Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py
==============================================================================
--- pypy/dist/pypy/translator/cli/test/test_builtin.py (original)
+++ pypy/dist/pypy/translator/cli/test/test_builtin.py Thu Aug 3 14:55:11 2006
@@ -20,18 +20,24 @@
def test_os_read(self):
BaseTestRbuiltin.test_os_read(self)
- def test_os_read_binary(self):
+ def test_os_read_binary_crlf(self):
tmpfile = str(udir.udir.join("os_read_test"))
- def fn():
+ def fn(flag):
from pypy.module.__builtin__.importing import BIN_READMASK
- fd = os.open(tmpfile, BIN_READMASK, 0666)
+ if flag:
+ fd = os.open(tmpfile, BIN_READMASK, 0666)
+ else:
+ fd = os.open(tmpfile, os.O_RDONLY, 0666)
res = os.read(fd, 4096)
os.close(fd)
- return len(res)
+ return res
f = file(tmpfile, 'w')
f.write('Hello\nWorld')
f.close()
- assert self.interpret(fn, []) == len(file(tmpfile, 'rb').read())
+ res = self.ll_to_string(self.interpret(fn, [True]))
+ assert res == file(tmpfile, 'rb').read()
+ res = self.ll_to_string(self.interpret(fn, [False]))
+ assert res == file(tmpfile, 'r').read()
# the following tests can't be executed with gencli because they
# returns file descriptors, and cli code is executed in another
@@ -45,6 +51,7 @@
pass
def test_os_open_write(self):
+ skip_win()
tmpdir = str(udir.udir.join("os_write_test"))
def fn():
fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT, 0777)
@@ -53,6 +60,28 @@
self.interpret(fn, [])
assert file(tmpdir).read() == 'hello world'
+ def test_os_write_magic(self):
+ from pypy.module.__builtin__.importing import BIN_WRITEMASK
+ skip_win()
+ MAGIC = 62061 | (ord('\r')<<16) | (ord('\n')<<24)
+ tmpfile = str(udir.udir.join("os_write_test"))
+ def long2str(x):
+ a = x & 0xff
+ x >>= 8
+ b = x & 0xff
+ x >>= 8
+ c = x & 0xff
+ x >>= 8
+ d = x & 0xff
+ return chr(a) + chr(b) + chr(c) + chr(d)
+ def fn(magic):
+ fd = os.open(tmpfile, BIN_WRITEMASK, 0777)
+ os.write(fd, long2str(magic))
+ os.close(fd)
+ self.interpret(fn, [MAGIC])
+ contents = file(tmpfile).read()
+ assert contents == long2str(MAGIC)
+
def test_os_stat(self):
def fn():
return os.stat('.')[0]
More information about the Pypy-commit
mailing list