[pypy-svn] r69264 - in pypy/branch/io-lang/pypy/lang/io: . test

david at codespeak.net david at codespeak.net
Fri Nov 13 15:20:05 CET 2009


Author: david
Date: Fri Nov 13 15:20:04 2009
New Revision: 69264

Added:
   pypy/branch/io-lang/pypy/lang/io/compiler.py
   pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
Modified:
   pypy/branch/io-lang/pypy/lang/io/objspace.py
Log:
Add Compiler object and implement tokensForString method to provide access to the lexer from io level

Added: pypy/branch/io-lang/pypy/lang/io/compiler.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/compiler.py	Fri Nov 13 15:20:04 2009
@@ -0,0 +1,19 @@
+from pypy.lang.io.register import register_method
+from pypy.lang.io.model import W_Object, W_Number
+from pypy.lang.io.parser import get_lexer
+ at register_method('Compiler', 'tokensForString')
+def compiler_tokens_for_string(space, w_target, w_message, w_context):
+    input = w_message.arguments[0].eval(space, w_context, w_target).value
+    io_tokens = []
+    for token in get_lexer().tokenize(input):
+        t = W_Object(space)
+        if token.source in ['[', '{'] and len(io_tokens) > 0:
+            io_tokens[-1].slots['character'].value = token.source_pos.columnno
+            
+        t.slots['character'] = W_Number(space, len(token.source) + token.source_pos.columnno)
+        t.slots['line'] = W_Number(space, token.source_pos.lineno + 1)
+        t.slots['type'] = space.w_sequence.clone_and_init(token.name)
+        t.slots['name'] = space.w_sequence.clone_and_init(token.source)
+        io_tokens.append(t)
+    
+    return space.w_list.clone_and_init(space, io_tokens)
\ No newline at end of file

Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/objspace.py	(original)
+++ pypy/branch/io-lang/pypy/lang/io/objspace.py	Fri Nov 13 15:20:04 2009
@@ -12,6 +12,7 @@
 import pypy.lang.io.map
 import pypy.lang.io.coroutine
 import pypy.lang.io.sequence
+import pypy.lang.io.compiler
 
 class ObjSpace(object):
     """docstring for ObjSpace"""
@@ -29,6 +30,7 @@
         self.w_call = W_Object(self, [self.w_object])
         self.w_map = W_Map(self, [self.w_object])
         self.w_coroutine = W_Coroutine.w_getmain(self)
+        self.w_compiler = W_Object(self, [self.w_object])
         # flow control objects
         self.w_normal = W_Object(self, [self.w_object])
         self.w_break = W_Object(self, [self.w_object])
@@ -42,6 +44,7 @@
         # default stop state
         self.stop_status = self.w_normal
         self.w_return_value = self.w_nil
+        
         self.init_w_object()        
         
         self.init_w_protos()
@@ -57,7 +60,6 @@
         
         self.init_w_number()
         
-        
         self.init_w_core()
         
         self.init_w_call()
@@ -72,6 +74,7 @@
         
         self.init_stored_messages()
 
+        self.init_w_compiler()
         
     def init_w_map(self):
         for key, function in cfunction_definitions['Map'].items():
@@ -109,6 +112,7 @@
         self.w_core.slots['Number'] = self.w_number
         self.w_core.slots['Sequence'] = self.w_sequence
         self.w_core.slots['ImmutableSequence'] = self.w_immutable_sequence
+        self.w_core.slots['Compiler'] = self.w_compiler
 
     def init_w_number(self):
         self.w_number = instantiate(W_Number)
@@ -157,7 +161,11 @@
     	self.w_core.slots['Eol'] = self.w_eol
         self.w_core.slots['Eol'].slots['type'] = W_ImmutableSequence(self, 'Eol')
         
-        
+    def init_w_compiler(self):
+        self.w_compiler.slots['type'] = W_ImmutableSequence(self, 'Compiler')
+        for key, function in cfunction_definitions['Compiler'].items():
+            self.w_compiler.slots[key] = W_CFunction(self, function)
+
     def init_w_coroutine(self):
         for key, function in cfunction_definitions['Coroutine'].items():
             self.w_coroutine.slots[key] = W_CFunction(self, function)

Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py	Fri Nov 13 15:20:04 2009
@@ -0,0 +1,170 @@
+from pypy.lang.io.parserhack import interpret
+from pypy.lang.io.model import W_Object, W_Number
+import py
+
+def test_compiler_is_present():
+    inp = "Compiler"
+    res, space = interpret(inp)
+    assert isinstance(res, W_Object)
+    assert res.slots['type'].value == 'Compiler'
+    assert res.protos == [space.w_object]
+    
+def test_compiler_token_for_number_is_number():
+    inp = 'Compiler tokensForString("1")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 1
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Number'
+    assert res.items[0].slots['name'].value == "1"
+    assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_HexNumber():
+    inp = 'Compiler tokensForString("0x1")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 3
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'HexNumber'
+    assert res.items[0].slots['name'].value == "0x1"
+    assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_open_paren():
+    inp = 'Compiler tokensForString("()")'
+    res, space = interpret(inp)
+    assert res.items[1].slots['character'].value == 1
+    assert res.items[1].slots['line'].value == 1
+    assert res.items[1].slots['type'].value == 'OpenParen'
+    assert res.items[1].slots['name'].value == "("
+    assert isinstance(res.items[1], W_Object)
+    
+def test_compiler_token_close_paren():
+    inp = 'Compiler tokensForString("()")'
+    res, space = interpret(inp)
+    assert res.items[2].slots['character'].value == 2
+    assert res.items[2].slots['line'].value == 1
+    assert res.items[2].slots['type'].value == 'CloseParen'
+    assert res.items[2].slots['name'].value == ")"
+    assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+    inp = 'Compiler tokensForString(" ()")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 1
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Identifier'
+    assert res.items[0].slots['name'].value == ""
+    assert isinstance(res.items[0], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+    inp = 'Compiler tokensForString("()")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 0
+    assert res.items[0].slots['line'].value == 1
+    assert isinstance(res.items[0], W_Object)
+
+# curlyBraces
+def test_compiler_token_open_squareBrackets():
+    inp = 'Compiler tokensForString("[]")'
+    res, space = interpret(inp)
+    assert res.items[1].slots['character'].value == 1
+    assert res.items[1].slots['line'].value == 1
+    assert res.items[1].slots['type'].value == 'OpenParen'
+    assert res.items[1].slots['name'].value == "["
+    assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_squareBrackets():
+    inp = 'Compiler tokensForString("[]")'
+    res, space = interpret(inp)
+    assert res.items[2].slots['character'].value == 2
+    assert res.items[2].slots['line'].value == 1
+    assert res.items[2].slots['type'].value == 'CloseParen'
+    assert res.items[2].slots['name'].value == "]"
+    assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_squareBrackets_message():
+    inp = 'Compiler tokensForString("[]")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 0
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Identifier'
+    assert res.items[0].slots['name'].value == "squareBrackets"
+    assert isinstance(res.items[0], W_Object)
+
+# curlyBrackets
+def test_compiler_token_open_curlyBrackets():
+    inp = 'Compiler tokensForString("{}")'
+    res, space = interpret(inp)
+    assert res.items[1].slots['character'].value == 1
+    assert res.items[1].slots['line'].value == 1
+    assert res.items[1].slots['type'].value == 'OpenParen'
+    assert res.items[1].slots['name'].value == "{"
+    assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_curlyBrackets():
+    inp = 'Compiler tokensForString("{}")'
+    res, space = interpret(inp)
+    assert res.items[2].slots['character'].value == 2
+    assert res.items[2].slots['line'].value == 1
+    assert res.items[2].slots['type'].value == 'CloseParen'
+    assert res.items[2].slots['name'].value == "}"
+    assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_curlyBrackets_message():
+    inp = 'Compiler tokensForString("{}")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 0
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Identifier'
+    assert res.items[0].slots['name'].value == "curlyBrackets"
+    assert isinstance(res.items[0], W_Object)
+    
+def test_compiler_identifier_token():
+    inp = 'Compiler tokensForString("foo")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 3
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Identifier'
+    assert res.items[0].slots['name'].value == 'foo'
+    assert isinstance(res.items[0], W_Object)
+    
+
+def test_compiler_terminator_token():
+    inp = 'Compiler tokensForString(";")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['character'].value == 1
+    assert res.items[0].slots['line'].value == 1
+    assert res.items[0].slots['type'].value == 'Terminator'
+    assert res.items[0].slots['name'].value == ";"
+    assert isinstance(res.items[0], W_Object)
+    
+def test_compiler_comma_token():
+    inp = 'Compiler tokensForString("(1,2)")'
+    res, space = interpret(inp)
+    assert res.items[3].slots['character'].value == 3
+    assert res.items[3].slots['line'].value == 1
+    assert res.items[3].slots['type'].value == 'Comma'
+    assert res.items[3].slots['name'].value == ","
+    assert isinstance(res.items[3], W_Object)
+
+def test_compiler_triquote_token():
+    py.test.skip('Problem in the parserhack')
+    inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['type'].value == 'TriQuote'
+    assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\""
+    assert isinstance(res.items[0], W_Object)
+
+def test_compiler_monoquote_token():
+    py.test.skip('Problem in the parserhack')
+    inp = 'Compiler tokensForString("\\\"lorem\\\"")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['name'].value == "\"lorem\""
+    assert res.items[0].slots['type'].value == 'MonoQuote'
+    assert isinstance(res.items[0], W_Object)
+
+def test_compiler_comment_token():
+    py.test.skip("These Tokens are ignored by the lexer")
+    inp = 'Compiler tokensForString("xxx")'
+    res, space = interpret(inp)
+    assert res.items[0].slots['type'].value == 'Comment'
+    assert res.items[0].slots['name'].value == "??"
+    assert isinstance(res.items[0], space.x)
\ No newline at end of file



More information about the Pypy-commit mailing list