[pypy-svn] r36911 - in pypy/branch/ast-experiments/pypy/module/recparser: . test
syt at codespeak.net
syt at codespeak.net
Thu Jan 18 10:05:46 CET 2007
Author: syt
Date: Thu Jan 18 10:05:45 2007
New Revision: 36911
Added:
pypy/branch/ast-experiments/pypy/module/recparser/test/test_dyn_grammarrules.py
Modified:
pypy/branch/ast-experiments/pypy/module/recparser/__init__.py
pypy/branch/ast-experiments/pypy/module/recparser/codegen.py
pypy/branch/ast-experiments/pypy/module/recparser/compat.py
pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
pypy/branch/ast-experiments/pypy/module/recparser/test/test_parser.py
Log:
Leysin sprint work:
more tests and bug fixes on the parser package
Modified: pypy/branch/ast-experiments/pypy/module/recparser/__init__.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/__init__.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/__init__.py Thu Jan 18 10:05:45 2007
@@ -48,8 +48,8 @@
'decode_string_literal': 'pyparser.decode_string_literal',
'install_compiler_hook' : 'pypy.interpreter.pycompiler.install_compiler_hook',
'insert_grammar_rule' : 'pypy.interpreter.pycompiler.insert_grammar_rule',
- 'rules' : 'pypy.interpreter.pyparser.pythonparse.grammar_rules',
- 'parse_grammar' : 'pypy.interpreter.pyparser.pythonparse.parse_grammar',
+ #'rules' : 'pypy.interpreter.pyparser.pythonparse.grammar_rules',
+ #'parse_grammar' : 'pypy.interpreter.pyparser.pythonparse.parse_grammar',
}
# Automatically exports each AST class
Modified: pypy/branch/ast-experiments/pypy/module/recparser/codegen.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/codegen.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/codegen.py Thu Jan 18 10:05:45 2007
@@ -54,7 +54,7 @@
def get_size(self):
s = 0
- for i in insns:
+ for i in self.insns:
s += i.size()
return s
Modified: pypy/branch/ast-experiments/pypy/module/recparser/compat.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/compat.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/compat.py Thu Jan 18 10:05:45 2007
@@ -1,12 +1,17 @@
"""Compatibility layer for CPython's parser module"""
-from pythonparse import parse_python_source
-from pythonutil import PYTHON_PARSER
+from pypy.interpreter.pyparser.tuplebuilder import TupleBuilder
+from pythonparse import make_pyparser
+from pythonutil import pypy_parse
+import symbol # XXX use PYTHON_PARSER.symbols ?
from compiler import transformer, compile as pycompile
+PYTHON_PARSER = make_pyparser()
+
def suite( source ):
strings = [line+'\n' for line in source.split('\n')]
- builder = parse_python_source( strings, PYTHON_PARSER, "file_input" )
+ builder = TupleBuilder(PYTHON_PARSER)
+ PYTHON_PARSER.parse_source(source, 'exec', builder)
nested_tuples = builder.stack[-1].as_tuple()
if builder.source_encoding is not None:
return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
@@ -16,7 +21,8 @@
def expr( source ):
strings = [line+'\n' for line in source.split('\n')]
- builder = parse_python_source( strings, PYTHON_PARSER, "eval_input" )
+ builder = TupleBuilder(PYTHON_PARSER)
+ PYTHON_PARSER.parse_source(source, 'eval', builder)
nested_tuples = builder.stack[-1].as_tuple()
if builder.source_encoding is not None:
return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
Modified: pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py Thu Jan 18 10:05:45 2007
@@ -45,7 +45,7 @@
def visit_tokennode( self, node ):
space = self.space
- tokens = space.parser.tokens
+ tokens = space.default_compiler.parser.tokens
num = node.name
lineno = node.lineno
if node.value is not None:
@@ -53,7 +53,7 @@
else:
if num not in ( tokens['NEWLINE'], tokens['INDENT'],
tokens['DEDENT'], tokens['ENDMARKER'] ):
- val = space.parser.tok_values[num]
+ val = space.default_compiler.parser.tok_rvalues[num]
else:
val = node.value or ''
if self.line_info:
@@ -148,11 +148,11 @@
totuple = interp2app(STType.descr_totuple),
)
-def parse_python_source(space, source, goal):
- builder = grammar.BaseGrammarBuilder(debug=False, rules=PYTHON_PARSER.rules)
+def parse_python_source(space, source, mode):
+ builder = grammar.BaseGrammarBuilder(debug=False, parser=PYTHON_PARSER)
builder.space = space
try:
- PYTHON_PARSER.parse_source(source, goal, builder )
+ PYTHON_PARSER.parse_source(source, mode, builder )
return builder.stack[-1]
except SyntaxError, e:
raise OperationError(space.w_SyntaxError,
@@ -160,14 +160,14 @@
def suite( space, source ):
# make the annotator life easier (don't use str.splitlines())
- syntaxtree = parse_python_source( space, source, "file_input" )
+ syntaxtree = parse_python_source( space, source, "exec" )
return space.wrap( STType(space, syntaxtree) )
suite.unwrap_spec = [ObjSpace, str]
def expr( space, source ):
# make the annotator life easier (don't use str.splitlines())
- syntaxtree = parse_python_source( space, source, "eval_input" )
+ syntaxtree = parse_python_source( space, source, "eval" )
return space.wrap( STType(space, syntaxtree) )
expr.unwrap_spec = [ObjSpace, str]
@@ -204,11 +204,8 @@
def source2ast(space, source):
- from pypy.interpreter.pyparser.pythonutil import AstBuilder, PYTHON_PARSER
- builder = AstBuilder(parser=PYTHON_PARSER, space=space)
- PYTHON_PARSER.parse_source(source, 'file_input', builder)
- ast_tree = builder.rule_stack[-1]
- return space.wrap(ast_tree)
+ from pypy.interpreter.pyparser.pythonutil import source2ast
+ return space.wrap(source2ast(source, 'exec', space=space))
source2ast.unwrap_spec = [ObjSpace, str]
Added: pypy/branch/ast-experiments/pypy/module/recparser/test/test_dyn_grammarrules.py
==============================================================================
--- (empty file)
+++ pypy/branch/ast-experiments/pypy/module/recparser/test/test_dyn_grammarrules.py Thu Jan 18 10:05:45 2007
@@ -0,0 +1,61 @@
+class AppTest_InsertGrammarRules:
+
+ def test_do_while(self):
+ import parser
+
+ newrules = """
+ compound_stmt: if_stmt | on_stmt | unless_stmt | dountil_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
+ dountil_stmt: 'do' ':' suite 'until' test
+ unless_stmt: 'unless' test ':' suite
+ on_stmt: 'on' NAME '=' test ':' suite ['else' ':' suite]
+ """
+
+ def build_dountil_stmt(items):
+ """ 'do' ':' suite 'until' ':' test """
+ lineno = items[0].lineno
+ suite = items[2]
+ test = items[-1]
+ while_stmt = parser.ASTWhile(parser.ASTNot(test), suite, None, lineno)
+ return parser.ASTStmt([suite, while_stmt], lineno)
+
+ def build_unless_stmt(its):
+ """ 'unless' test ':' suite """
+ lineno = its[0].lineno
+ return parser.ASTIf([(parser.ASTNot(its[1]), its[3])], None, lineno)
+
+ def make_assignment(var, node):
+ # XXX: consts.OP_APPLY
+ return parser.ASTAssign([parser.ASTAssName('x', 0)], node)
+
+ def build_on_stmt(items):
+ """ 'on' NAME = test ':' suite 'else' ':' suite"""
+ varname = items[1].value
+ test = items[3]
+ suite = items[5]
+ assign = make_assignment(varname, test)
+ if len(items) == 9:
+ else_ = items[-1]
+ else:
+ else_ = None
+ test = parser.ASTIf([(parser.ASTName(varname), suite)], else_, items[0].lineno)
+ return parser.ASTStmt([assign, test], items[0].lineno)
+
+ parser.insert_grammar_rule(newrules, {'dountil_stmt' : build_dountil_stmt,
+ 'unless_stmt': build_unless_stmt,
+ 'on_stmt' : build_on_stmt,
+ })
+
+ # now we should be able to use do...until and unless statements
+ d = {}
+ exec '''
+a = 0
+do:
+ a += 1
+until True
+
+b = 0
+unless a == 2: b = 3
+ ''' in d
+ assert d['a'] == 1
+ assert d['b'] == 3
+
Modified: pypy/branch/ast-experiments/pypy/module/recparser/test/test_parser.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/test/test_parser.py (original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/test/test_parser.py Thu Jan 18 10:05:45 2007
@@ -15,3 +15,8 @@
def test_enc_minimal(self):
import parser
parser.suite("# -*- coding: koi8-u -*-*\ngreat()")
+
+ def test_simple_ass_totuple(self):
+ import parser
+ parser.suite("a = 3").totuple()
+
More information about the Pypy-commit
mailing list