[pypy-svn] r69264 - in pypy/branch/io-lang/pypy/lang/io: . test
david at codespeak.net
david at codespeak.net
Fri Nov 13 15:20:05 CET 2009
Author: david
Date: Fri Nov 13 15:20:04 2009
New Revision: 69264
Added:
pypy/branch/io-lang/pypy/lang/io/compiler.py
pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
Modified:
pypy/branch/io-lang/pypy/lang/io/objspace.py
Log:
Add Compiler object and implement tokensForString method to provide access to the lexer from io level
Added: pypy/branch/io-lang/pypy/lang/io/compiler.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/compiler.py Fri Nov 13 15:20:04 2009
@@ -0,0 +1,19 @@
+from pypy.lang.io.register import register_method
+from pypy.lang.io.model import W_Object, W_Number
+from pypy.lang.io.parser import get_lexer
+ at register_method('Compiler', 'tokensForString')
+def compiler_tokens_for_string(space, w_target, w_message, w_context):
+ input = w_message.arguments[0].eval(space, w_context, w_target).value
+ io_tokens = []
+ for token in get_lexer().tokenize(input):
+ t = W_Object(space)
+ if token.source in ['[', '{'] and len(io_tokens) > 0:
+ io_tokens[-1].slots['character'].value = token.source_pos.columnno
+
+ t.slots['character'] = W_Number(space, len(token.source) + token.source_pos.columnno)
+ t.slots['line'] = W_Number(space, token.source_pos.lineno + 1)
+ t.slots['type'] = space.w_sequence.clone_and_init(token.name)
+ t.slots['name'] = space.w_sequence.clone_and_init(token.source)
+ io_tokens.append(t)
+
+ return space.w_list.clone_and_init(space, io_tokens)
\ No newline at end of file
Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/objspace.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/objspace.py Fri Nov 13 15:20:04 2009
@@ -12,6 +12,7 @@
import pypy.lang.io.map
import pypy.lang.io.coroutine
import pypy.lang.io.sequence
+import pypy.lang.io.compiler
class ObjSpace(object):
"""docstring for ObjSpace"""
@@ -29,6 +30,7 @@
self.w_call = W_Object(self, [self.w_object])
self.w_map = W_Map(self, [self.w_object])
self.w_coroutine = W_Coroutine.w_getmain(self)
+ self.w_compiler = W_Object(self, [self.w_object])
# flow control objects
self.w_normal = W_Object(self, [self.w_object])
self.w_break = W_Object(self, [self.w_object])
@@ -42,6 +44,7 @@
# default stop state
self.stop_status = self.w_normal
self.w_return_value = self.w_nil
+
self.init_w_object()
self.init_w_protos()
@@ -57,7 +60,6 @@
self.init_w_number()
-
self.init_w_core()
self.init_w_call()
@@ -72,6 +74,7 @@
self.init_stored_messages()
+ self.init_w_compiler()
def init_w_map(self):
for key, function in cfunction_definitions['Map'].items():
@@ -109,6 +112,7 @@
self.w_core.slots['Number'] = self.w_number
self.w_core.slots['Sequence'] = self.w_sequence
self.w_core.slots['ImmutableSequence'] = self.w_immutable_sequence
+ self.w_core.slots['Compiler'] = self.w_compiler
def init_w_number(self):
self.w_number = instantiate(W_Number)
@@ -157,7 +161,11 @@
self.w_core.slots['Eol'] = self.w_eol
self.w_core.slots['Eol'].slots['type'] = W_ImmutableSequence(self, 'Eol')
-
+ def init_w_compiler(self):
+ self.w_compiler.slots['type'] = W_ImmutableSequence(self, 'Compiler')
+ for key, function in cfunction_definitions['Compiler'].items():
+ self.w_compiler.slots[key] = W_CFunction(self, function)
+
def init_w_coroutine(self):
for key, function in cfunction_definitions['Coroutine'].items():
self.w_coroutine.slots[key] = W_CFunction(self, function)
Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py Fri Nov 13 15:20:04 2009
@@ -0,0 +1,170 @@
+from pypy.lang.io.parserhack import interpret
+from pypy.lang.io.model import W_Object, W_Number
+import py
+
+def test_compiler_is_present():
+ inp = "Compiler"
+ res, space = interpret(inp)
+ assert isinstance(res, W_Object)
+ assert res.slots['type'].value == 'Compiler'
+ assert res.protos == [space.w_object]
+
+def test_compiler_token_for_number_is_number():
+ inp = 'Compiler tokensForString("1")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Number'
+ assert res.items[0].slots['name'].value == "1"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_HexNumber():
+ inp = 'Compiler tokensForString("0x1")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 3
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'HexNumber'
+ assert res.items[0].slots['name'].value == "0x1"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_open_paren():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "("
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_close_paren():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == ")"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+ inp = 'Compiler tokensForString(" ()")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == ""
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert isinstance(res.items[0], W_Object)
+
+# curlyBraces
+def test_compiler_token_open_squareBrackets():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "["
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_squareBrackets():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == "]"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_squareBrackets_message():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == "squareBrackets"
+ assert isinstance(res.items[0], W_Object)
+
+# curlyBrackets
+def test_compiler_token_open_curlyBrackets():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "{"
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_curlyBrackets():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == "}"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_curlyBrackets_message():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == "curlyBrackets"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_identifier_token():
+ inp = 'Compiler tokensForString("foo")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 3
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == 'foo'
+ assert isinstance(res.items[0], W_Object)
+
+
+def test_compiler_terminator_token():
+ inp = 'Compiler tokensForString(";")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Terminator'
+ assert res.items[0].slots['name'].value == ";"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_comma_token():
+ inp = 'Compiler tokensForString("(1,2)")'
+ res, space = interpret(inp)
+ assert res.items[3].slots['character'].value == 3
+ assert res.items[3].slots['line'].value == 1
+ assert res.items[3].slots['type'].value == 'Comma'
+ assert res.items[3].slots['name'].value == ","
+ assert isinstance(res.items[3], W_Object)
+
+def test_compiler_triquote_token():
+ py.test.skip('Problem in the parserhack')
+ inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['type'].value == 'TriQuote'
+ assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\""
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_monoquote_token():
+ py.test.skip('Problem in the parserhack')
+ inp = 'Compiler tokensForString("\\\"lorem\\\"")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['name'].value == "\"lorem\""
+ assert res.items[0].slots['type'].value == 'MonoQuote'
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_comment_token():
+ py.test.skip("These Tokens are ignored by the lexer")
+ inp = 'Compiler tokensForString("xxx")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['type'].value == 'Comment'
+ assert res.items[0].slots['name'].value == "??"
+ assert isinstance(res.items[0], space.x)
\ No newline at end of file
More information about the Pypy-commit
mailing list