[pypy-svn] r69446 - in pypy/branch/io-lang/pypy/lang/io: . test
david at codespeak.net
david at codespeak.net
Thu Nov 19 15:30:02 CET 2009
Author: david
Date: Thu Nov 19 15:30:01 2009
New Revision: 69446
Added:
pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py
pypy/branch/io-lang/pypy/lang/io/test/test_parser.py
Modified:
pypy/branch/io-lang/pypy/lang/io/compiler.py
pypy/branch/io-lang/pypy/lang/io/model.py
pypy/branch/io-lang/pypy/lang/io/parser.py
pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py
Log:
Parser implementation, without error messages and operator shuffling
Modified: pypy/branch/io-lang/pypy/lang/io/compiler.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/compiler.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/compiler.py Thu Nov 19 15:30:01 2009
@@ -1,6 +1,6 @@
from pypy.lang.io.register import register_method
from pypy.lang.io.model import W_Object, W_Number
-from pypy.lang.io.parser import get_lexer
+from pypy.lang.io.parser import get_lexer, parse
@register_method('Compiler', 'tokensForString')
def compiler_tokens_for_string(space, w_target, w_message, w_context):
input = w_message.arguments[0].eval(space, w_context, w_target).value
@@ -9,11 +9,14 @@
t = W_Object(space)
if token.source in ['[', '{'] and len(io_tokens) > 0:
io_tokens[-1].slots['character'].value = token.source_pos.columnno
-
t.slots['character'] = W_Number(space, len(token.source) + token.source_pos.columnno)
t.slots['line'] = W_Number(space, token.source_pos.lineno + 1)
t.slots['type'] = space.w_sequence.clone_and_init(token.name)
t.slots['name'] = space.w_sequence.clone_and_init(token.source)
io_tokens.append(t)
- return space.w_list.clone_and_init(space, io_tokens)
\ No newline at end of file
+ return space.w_list.clone_and_init(space, io_tokens)
+
+ at register_method('Compiler', 'messageForString', unwrap_spec=[object, str])
+def compiler_message_for_string(space, w_target, string):
+ return parse(space, string)
\ No newline at end of file
Modified: pypy/branch/io-lang/pypy/lang/io/model.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/model.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/model.py Thu Nov 19 15:30:01 2009
@@ -292,4 +292,4 @@
except ValueError:
pass
if literal.startswith('"') and literal.endswith('"'):
- return space.w_immutable_sequence.clone_and_init(literal[1:-1])
\ No newline at end of file
+ return space.w_sequence.clone_and_init(literal[1:-1])
\ No newline at end of file
Modified: pypy/branch/io-lang/pypy/lang/io/parser.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/parser.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/parser.py Thu Nov 19 15:30:01 2009
@@ -1,5 +1,10 @@
+# TODO:
+# set line and character number on messages
+# operator shuffling
+
from pypy.rlib.parsing.regexparse import parse_regex
from pypy.rlib.parsing.lexer import Lexer, Token, SourcePos
+from pypy.lang.io.model import W_Message, parse_literal, parse_hex
# taken from rlib/parsing/test/python_lexer.py
# reg exp helper methods
@@ -28,7 +33,7 @@
Hexnumber = r'0[xX][0-9a-fA-F]*'
# Comments and Whitespace, the ignored stuff
-Whitespace = r'[ \f\t\n]*'
+Whitespace = r'[ \f\t]*'
slashStarComment = r'/\*[^*/]*\*/'
slashSlashComment = r'//[^\n]*\n'
poundComment = r'#[^\n]*\n'
@@ -43,7 +48,7 @@
MonoQuote = r'"([^"]|(\\"))*"'
# ;
-Terminator = r'(;'+maybe(Whitespace)+')+'
+Terminator = r'([\n;]'+maybe(Whitespace)+')+'
# ,
Comma = r'\,'
@@ -72,6 +77,8 @@
i = -1
while(i < len(tokens)-1):
i += 1
+ if tokens[i].name == 'Terminator':
+ tokens[i].source = ';'
if tokens[i].name != 'OpenParen':
continue
if tokens[i].source == '(' and (i == 0 or tokens[i-1].name != 'Identifier'):
@@ -89,4 +96,58 @@
tokens[i].source_pos.columnno)))
i += 1
- return tokens
\ No newline at end of file
+ return tokens
+
+def parse(space, string):
+ return IoParser(string, space).parse()
+
+class IoParser(object):
+ def __init__(self, code, space):
+ super(IoParser, self).__init__()
+ self.code = code
+ self.space = space
+ self.tokens = get_lexer().tokenize(self.code)
+ if len(self.tokens) > 0 and self.tokens[0].name == 'Terminator':
+ self.tokens.pop(0)
+ if len(self.tokens) > 0 and self.tokens[-1].name == 'Terminator':
+ self.tokens.pop()
+
+
+ def parse(self):
+ if len(self.tokens) == 0:
+ return W_Message(self.space, 'nil', [])
+ token = self.tokens.pop(0)
+ # method = getattr(self, "parse_" + token.name.lower())
+ arguments = self.parse_arguments()
+ message = self.parse_token(token, arguments)
+ message.next = self.parse_next()
+ return message
+
+ def parse_next(self):
+ if len(self.tokens) > 0 and self.tokens[0].name not in ['Comma', 'OpenParen', 'CloseParen']:
+ return self.parse()
+ else:
+ return None
+
+ def parse_arguments(self):
+ if len(self.tokens) > 0 and self.tokens[0].name == 'OpenParen':
+ arguments = []
+ t = self.tokens.pop(0)
+ assert t.name == 'OpenParen'
+
+ while self.tokens[0].name != 'CloseParen':
+ if self.tokens[0].name == 'Comma':
+ self.tokens.pop(0)
+ arguments.append(self.parse())
+
+ t = self.tokens.pop(0)
+ assert t.name == 'CloseParen'
+ else:
+ arguments = []
+ return arguments
+
+ def parse_token(self, token, args=[]):
+ m = W_Message(self.space, token.source, args)
+ if token.name != 'Identifier':
+ m.cached_result = parse_literal(self.space, token.source)
+ return m
Modified: pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py Thu Nov 19 15:30:01 2009
@@ -1,5 +1,5 @@
from pypy.lang.io.parserhack import interpret
-from pypy.lang.io.model import W_Object, W_Number
+from pypy.lang.io.model import W_Object, W_Number, W_Message
import py
def test_compiler_is_present():
@@ -8,163 +8,70 @@
assert isinstance(res, W_Object)
assert res.slots['type'].value == 'Compiler'
assert res.protos == [space.w_object]
-
-def test_compiler_token_for_number_is_number():
- inp = 'Compiler tokensForString("1")'
- res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 1
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Number'
- assert res.items[0].slots['name'].value == "1"
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_token_HexNumber():
- inp = 'Compiler tokensForString("0x1")'
- res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 3
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'HexNumber'
- assert res.items[0].slots['name'].value == "0x1"
- assert isinstance(res.items[0], W_Object)
-def test_compiler_token_open_paren():
- inp = 'Compiler tokensForString("()")'
+def test_parse_number():
+ inp = 'Compiler messageForString("42")'
res, space = interpret(inp)
- assert res.items[1].slots['character'].value == 1
- assert res.items[1].slots['line'].value == 1
- assert res.items[1].slots['type'].value == 'OpenParen'
- assert res.items[1].slots['name'].value == "("
- assert isinstance(res.items[1], W_Object)
-
-def test_compiler_token_close_paren():
- inp = 'Compiler tokensForString("()")'
- res, space = interpret(inp)
- assert res.items[2].slots['character'].value == 2
- assert res.items[2].slots['line'].value == 1
- assert res.items[2].slots['type'].value == 'CloseParen'
- assert res.items[2].slots['name'].value == ")"
- assert isinstance(res.items[2], W_Object)
-
-def test_compiler_parse_paren_produces_anon_message():
- inp = 'Compiler tokensForString(" ()")'
- res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 1
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Identifier'
- assert res.items[0].slots['name'].value == ""
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_parse_paren_produces_anon_message():
- inp = 'Compiler tokensForString("()")'
- res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 0
- assert res.items[0].slots['line'].value == 1
- assert isinstance(res.items[0], W_Object)
-
-# curlyBraces
-def test_compiler_token_open_squareBrackets():
- inp = 'Compiler tokensForString("[]")'
- res, space = interpret(inp)
- assert res.items[1].slots['character'].value == 1
- assert res.items[1].slots['line'].value == 1
- assert res.items[1].slots['type'].value == 'OpenParen'
- assert res.items[1].slots['name'].value == "["
- assert isinstance(res.items[1], W_Object)
-
-def test_compiler_token_squareBrackets():
- inp = 'Compiler tokensForString("[]")'
+ assert isinstance(res, W_Message)
+ assert res.name == '42'
+ assert isinstance(res.cached_result, W_Number)
+ assert res.cached_result.value == 42
+
+def test_parse_hexnumber():
+ inp = 'Compiler messageForString("0xf")'
res, space = interpret(inp)
- assert res.items[2].slots['character'].value == 2
- assert res.items[2].slots['line'].value == 1
- assert res.items[2].slots['type'].value == 'CloseParen'
- assert res.items[2].slots['name'].value == "]"
- assert isinstance(res.items[2], W_Object)
-
-def test_compiler_parse_paren_produces_squareBrackets_message():
- inp = 'Compiler tokensForString("[]")'
+ assert isinstance(res, W_Message)
+ assert res.name == '0xf'
+ assert isinstance(res.cached_result, W_Number)
+ assert res.cached_result.value == 15
+
+def test_parse_message_with_artguments():
+ inp = 'Compiler messageForString("a(1,2)")'
res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 0
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Identifier'
- assert res.items[0].slots['name'].value == "squareBrackets"
- assert isinstance(res.items[0], W_Object)
-
-# curlyBrackets
-def test_compiler_token_open_curlyBrackets():
- inp = 'Compiler tokensForString("{}")'
- res, space = interpret(inp)
- assert res.items[1].slots['character'].value == 1
- assert res.items[1].slots['line'].value == 1
- assert res.items[1].slots['type'].value == 'OpenParen'
- assert res.items[1].slots['name'].value == "{"
- assert isinstance(res.items[1], W_Object)
-
-def test_compiler_token_curlyBrackets():
- inp = 'Compiler tokensForString("{}")'
+ assert isinstance(res, W_Message)
+ assert res.name == 'a'
+ assert len(res.arguments) == 2
+ assert res.arguments[0].name == '1'
+ assert res.arguments[1].name == '2'
+
+def test_parse_message_chain():
+ inp = 'Compiler messageForString("1 2")'
res, space = interpret(inp)
- assert res.items[2].slots['character'].value == 2
- assert res.items[2].slots['line'].value == 1
- assert res.items[2].slots['type'].value == 'CloseParen'
- assert res.items[2].slots['name'].value == "}"
- assert isinstance(res.items[2], W_Object)
-
-def test_compiler_parse_paren_produces_curlyBrackets_message():
- inp = 'Compiler tokensForString("{}")'
+ assert isinstance(res, W_Message)
+ assert res.name == '1'
+ assert isinstance(res.next, W_Message)
+ assert res.next.name == '2'
+ assert res.next.cached_result.value == 2
+
+def test_parse_longer_message_chain():
+ inp = 'Compiler messageForString("1 2 3 4 5 6")'
res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 0
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Identifier'
- assert res.items[0].slots['name'].value == "curlyBrackets"
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_identifier_token():
- inp = 'Compiler tokensForString("foo")'
- res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 3
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Identifier'
- assert res.items[0].slots['name'].value == 'foo'
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_terminator_token():
- inp = 'Compiler tokensForString(";")'
+ assert isinstance(res, W_Message)
+ assert res.name == '1'
+
+ n = res.next
+ assert isinstance(n, W_Message)
+ assert n.name == '2'
+
+ n = n.next
+ assert isinstance(n, W_Message)
+ assert n.name == '3'
+
+ n = n.next
+ assert isinstance(n, W_Message)
+ assert n.name == '4'
+
+ n = n.next
+ assert isinstance(n, W_Message)
+ assert n.name == '5'
+
+ n = n.next
+ assert isinstance(n, W_Message)
+ assert n.name == '6'
+
+def test_parse_empty_input():
+ inp = 'Compiler messageForString("")'
res, space = interpret(inp)
- assert res.items[0].slots['character'].value == 1
- assert res.items[0].slots['line'].value == 1
- assert res.items[0].slots['type'].value == 'Terminator'
- assert res.items[0].slots['name'].value == ";"
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_comma_token():
- inp = 'Compiler tokensForString("(1,2)")'
- res, space = interpret(inp)
- assert res.items[3].slots['character'].value == 3
- assert res.items[3].slots['line'].value == 1
- assert res.items[3].slots['type'].value == 'Comma'
- assert res.items[3].slots['name'].value == ","
- assert isinstance(res.items[3], W_Object)
-
-def test_compiler_triquote_token():
- py.test.skip('Problem in the parserhack')
- inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")'
- res, space = interpret(inp)
- assert res.items[0].slots['type'].value == 'TriQuote'
- assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\""
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_monoquote_token():
- py.test.skip('Problem in the parserhack')
- inp = 'Compiler tokensForString("\\\"lorem\\\"")'
- res, space = interpret(inp)
- assert res.items[0].slots['name'].value == "\"lorem\""
- assert res.items[0].slots['type'].value == 'MonoQuote'
- assert isinstance(res.items[0], W_Object)
-
-def test_compiler_comment_token():
- py.test.skip("These Tokens are ignored by the lexer")
- inp = 'Compiler tokensForString("xxx")'
- res, space = interpret(inp)
- assert res.items[0].slots['type'].value == 'Comment'
- assert res.items[0].slots['name'].value == "??"
- assert isinstance(res.items[0], space.x)
\ No newline at end of file
+ assert isinstance(res, W_Message)
+ assert res.name == 'nil'
\ No newline at end of file
Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py Thu Nov 19 15:30:01 2009
@@ -0,0 +1,173 @@
+from pypy.lang.io.parserhack import interpret
+from pypy.lang.io.model import W_Object, W_Number
+import py
+
+def test_compiler_token_for_number_is_number():
+ inp = 'Compiler tokensForString("1")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Number'
+ assert res.items[0].slots['name'].value == "1"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_HexNumber():
+ inp = 'Compiler tokensForString("0x1")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 3
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'HexNumber'
+ assert res.items[0].slots['name'].value == "0x1"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_token_open_paren():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "("
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_close_paren():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == ")"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+ inp = 'Compiler tokensForString(" ()")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == ""
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_parse_paren_produces_anon_message():
+ inp = 'Compiler tokensForString("()")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert isinstance(res.items[0], W_Object)
+
+# curlyBraces
+def test_compiler_token_open_squareBrackets():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "["
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_squareBrackets():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == "]"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_squareBrackets_message():
+ inp = 'Compiler tokensForString("[]")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == "squareBrackets"
+ assert isinstance(res.items[0], W_Object)
+
+# curlyBrackets
+def test_compiler_token_open_curlyBrackets():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[1].slots['character'].value == 1
+ assert res.items[1].slots['line'].value == 1
+ assert res.items[1].slots['type'].value == 'OpenParen'
+ assert res.items[1].slots['name'].value == "{"
+ assert isinstance(res.items[1], W_Object)
+
+def test_compiler_token_curlyBrackets():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[2].slots['character'].value == 2
+ assert res.items[2].slots['line'].value == 1
+ assert res.items[2].slots['type'].value == 'CloseParen'
+ assert res.items[2].slots['name'].value == "}"
+ assert isinstance(res.items[2], W_Object)
+
+def test_compiler_parse_paren_produces_curlyBrackets_message():
+ inp = 'Compiler tokensForString("{}")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 0
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == "curlyBrackets"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_identifier_token():
+ inp = 'Compiler tokensForString("foo")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 3
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Identifier'
+ assert res.items[0].slots['name'].value == 'foo'
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_terminator_token_for_newline():
+ py.test.skip("Parserhack related")
+ inp = """Compiler tokensForString("\n")"""
+ res, space = interpret(inp)
+ # assert res.items[0].slots['character'].value == 1
+ # assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Terminator'
+ assert res.items[0].slots['name'].value == ";"
+ assert isinstance(res.items[0], W_Object)
+
+
+def test_compiler_terminator_token():
+ inp = 'Compiler tokensForString(";")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['character'].value == 1
+ assert res.items[0].slots['line'].value == 1
+ assert res.items[0].slots['type'].value == 'Terminator'
+ assert res.items[0].slots['name'].value == ";"
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_comma_token():
+ inp = 'Compiler tokensForString("(1,2)")'
+ res, space = interpret(inp)
+ assert res.items[3].slots['character'].value == 3
+ assert res.items[3].slots['line'].value == 1
+ assert res.items[3].slots['type'].value == 'Comma'
+ assert res.items[3].slots['name'].value == ","
+ assert isinstance(res.items[3], W_Object)
+
+def test_compiler_triquote_token():
+ py.test.skip('Problem in the parserhack')
+ inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['type'].value == 'TriQuote'
+ assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\""
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_monoquote_token():
+ py.test.skip('Problem in the parserhack')
+ inp = 'Compiler tokensForString("\\\"lorem\\\"")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['name'].value == "\"lorem\""
+ assert res.items[0].slots['type'].value == 'MonoQuote'
+ assert isinstance(res.items[0], W_Object)
+
+def test_compiler_comment_token():
+ py.test.skip("These Tokens are ignored by the lexer")
+ inp = 'Compiler tokensForString("xxx")'
+ res, space = interpret(inp)
+ assert res.items[0].slots['type'].value == 'Comment'
+ assert res.items[0].slots['name'].value == "??"
+ assert isinstance(res.items[0], space.x)
\ No newline at end of file
Modified: pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py
==============================================================================
--- pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py (original)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Thu Nov 19 15:30:01 2009
@@ -85,14 +85,20 @@
def test_lex_terminator1():
inp = ';;'
tokens = iolexer.tokenize(inp)
- assert tokens[0] == Token('Terminator', ';;', SourcePos(0, 0, 0))
+ assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0))
assert len(tokens) == 1
def test_lex_terminator2():
inp = '; ;'
tokens = iolexer.tokenize(inp)
- assert tokens[0] == Token('Terminator', '; ;', SourcePos(0, 0, 0))
+ assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0))
assert len(tokens) == 1
+
+def test_lex_newline_terminator():
+ inp = '\n'
+ tokens = iolexer.tokenize(inp)
+ assert len(tokens) == 1
+ assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0))
def test_lex_open_paren():
inp = '()'
Added: pypy/branch/io-lang/pypy/lang/io/test/test_parser.py
==============================================================================
--- (empty file)
+++ pypy/branch/io-lang/pypy/lang/io/test/test_parser.py Thu Nov 19 15:30:01 2009
@@ -0,0 +1,142 @@
+import py
+from pypy.lang.io.parser import parse, IoParser
+from pypy.lang.io.model import W_Message, W_Number
+from pypy.lang.io.objspace import ObjSpace
+
+space = ObjSpace()
+def test_parse_number_token():
+ t = parse(space, "4")
+ assert isinstance(t, W_Message)
+ assert t.name == '4'
+
+def test_parse_number_sets_cached_result():
+ t = parse(space, "4")
+ assert isinstance(t.cached_result, W_Number)
+ assert t.cached_result.value == 4
+
+def test_parse_hexnumber_token():
+ t = parse(space, "0xf")
+ assert isinstance(t, W_Message)
+ assert t.name == '0xf'
+
+def test_parse_hexnumber_sets_cached_result():
+ t = parse(space, '0xf')
+ assert isinstance(t.cached_result, W_Number)
+ assert t.cached_result.value == 15
+
+def test_parse_identifier():
+ t = parse(space, 'foo')
+ assert isinstance(t, W_Message)
+ assert t.name == 'foo'
+
+def test_parse_string():
+ t = parse(space, '"a"')
+ assert isinstance(t, W_Message)
+ assert t.name == '"a"'
+
+def test_parse_string_sets_cached_result():
+ t = parse(space, '"a"')
+ assert space.w_sequence in t.cached_result.protos
+ assert t.cached_result.value == 'a'
+
+def test_parse_tripple_quoted_string():
+ t = parse(space, '"""a"""')
+ assert isinstance(t, W_Message)
+ assert t.name == '"""a"""'
+
+def test_parse_tripple_quoted_string_sets_cached_result():
+ t = parse(space, '"a"')
+ assert space.w_sequence in t.cached_result.protos
+ assert t.cached_result.value == 'a'
+
+def test_parse_arguments_simple():
+ t = parse(space, 'a(1)')
+ assert len(t.arguments) == 1
+ assert t.arguments[0].name == '1'
+
+def test_parse_argument_list():
+ t = parse(space, 'a(1, "a", 0xa)')
+ assert len(t.arguments) == 3
+ assert t.arguments[0].name == '1'
+ assert t.arguments[1].name == '"a"'
+ assert t.arguments[2].name == '0xa'
+
+def test_parse_message_chain():
+ t = parse(space, 'a 1')
+ assert isinstance(t, W_Message)
+ assert t.name == 'a'
+ next = t.next
+ assert isinstance(next, W_Message)
+ assert next.name == '1'
+ assert next.cached_result.value == 1
+
+def test_parse_message_chain_with_arguments():
+ t = parse(space, 'a("foo", "bar") 1')
+ assert isinstance(t, W_Message)
+ assert t.name == 'a'
+ next = t.next
+ assert isinstance(next, W_Message)
+ assert next.name == '1'
+ assert next.cached_result.value == 1
+
+def test_parse_empty_string_produces_nil_message():
+ t = parse(space, '')
+ assert isinstance(t, W_Message)
+ assert t.name == 'nil'
+
+def test_parser_sets_line_and_char_no_on_message():
+ py.test.skip("Not implemented yet")
+
+def test_parse_only_terminator():
+ t = parse(space, ';')
+ assert isinstance(t, W_Message)
+ assert t.name == 'nil'
+
+def test_parse_only_terminator2():
+ t = parse(space, """\n""")
+ assert isinstance(t, W_Message)
+ assert t.name == 'nil'
+
+def test_parse_terminator_between_messages_appends_terminator_message():
+ t = parse(space, 'a ; b')
+
+ assert isinstance(t, W_Message)
+ assert t.name == 'a'
+ t1 = t.next
+
+ assert isinstance(t1, W_Message)
+ assert t1.name == ';'
+ t2 = t1.next
+
+ assert isinstance(t2, W_Message)
+ assert t2.name == 'b'
+
+def test_parse_terminator_between_messages_appends_terminator_message2():
+ t = parse(space, 'a ;;;;;; ; b')
+
+ assert isinstance(t, W_Message)
+ assert t.name == 'a'
+ t1 = t.next
+
+ assert isinstance(t1, W_Message)
+ assert t1.name == ';'
+ t2 = t1.next
+
+ assert isinstance(t2, W_Message)
+ assert t2.name == 'b'
+
+def test_parse_terminator_at_end_is_ignored():
+ t = parse(space, 'a ; b ;')
+
+ assert isinstance(t, W_Message)
+ assert t.name == 'a'
+ t1 = t.next
+
+ assert isinstance(t1, W_Message)
+ assert t1.name == ';'
+ t2 = t1.next
+
+ assert isinstance(t2, W_Message)
+ assert t2.name == 'b'
+
+ assert t2.next is None
\ No newline at end of file
More information about the Pypy-commit
mailing list