[pypy-svn] r66223 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler astcompiler/test pyparser pyparser/test

benjamin at codespeak.net benjamin at codespeak.net
Wed Jul 15 06:04:16 CEST 2009


Author: benjamin
Date: Wed Jul 15 06:04:13 2009
New Revision: 66223

Modified:
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py
   pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
hook the new compiler up to the interpreter

also provide a nice way, CompileInfo, to pass information like filename around


Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py	Wed Jul 15 06:04:13 2009
@@ -5,8 +5,8 @@
 from pypy.interpreter.pyparser import parsestring
 
 
-def ast_from_node(space, n):
-    return ASTBuilder(space, n).build_ast()
+def ast_from_node(space, n, compile_info):
+    return ASTBuilder(space, n, compile_info).build_ast()
 
 
 augassign_operator_map = {
@@ -41,13 +41,9 @@
 
 class ASTBuilder(object):
 
-    def __init__(self, space, n):
+    def __init__(self, space, n, compile_info):
         self.space = space
-        if n.type == syms.encoding_decl:
-            self.encoding = n.value
-            n = n.children[0]
-        else:
-            self.encoding = None
+        self.compile_info = compile_info
         self.root_node = n
 
     def build_ast(self):
@@ -103,7 +99,7 @@
             raise AssertionError("non-statement node")
 
     def error(self, msg, n):
-        raise SyntaxError(msg, n.lineno, n.column)
+        raise SyntaxError(msg, n.lineno, n.column, self.compile_info.filename)
 
     def check_forbidden_name(self, name, node):
         if name == "None":
@@ -1077,7 +1073,8 @@
                             first_child.lineno, first_child.column)
         elif first_child_type == tokens.STRING:
             space = self.space
-            sub_strings_w = [parsestring.parsestr(space, self.encoding, s.value)
+            encoding = self.compile_info.encoding
+            sub_strings_w = [parsestring.parsestr(space, encoding, s.value)
                              for s in atom_node.children]
             if len(sub_strings_w) > 1:
                 w_sub_strings = space.newlist(sub_strings_w)

Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py	Wed Jul 15 06:04:13 2009
@@ -253,7 +253,7 @@
 
 class SymtableBuilder(ast.GenericASTVisitor):
 
-    def __init__(self, space, module):
+    def __init__(self, space, module, compile_info):
         self.space = space
         self.module = module
         self.scopes = {}
@@ -263,8 +263,12 @@
         top = ModuleScope(module)
         self.globs = top.roles
         self.push_scope(top)
-        module.walkabout(self)
-        top.finalize(None, {}, {})
+        try:
+            module.walkabout(self)
+            top.finalize(None, {}, {})
+        except SyntaxError, e:
+            e.filename = compile_info.filename
+            raise
         self.pop_scope()
         assert not self.stack
 

Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py	Wed Jul 15 06:04:13 2009
@@ -16,8 +16,9 @@
         cls.parser = pyparse.PythonParser(cls.space)
 
     def get_ast(self, source, p_mode="exec"):
-        tree = self.parser.parse_source(source, p_mode)
-        ast_node = ast_from_node(self.space, tree)
+        info = pyparse.CompileInfo("<test>", p_mode)
+        tree = self.parser.parse_source(source, info)
+        ast_node = ast_from_node(self.space, tree, info)
         return ast_node
 
     def get_first_expr(self, source):
@@ -1019,9 +1020,10 @@
         assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
         sentence = u"Die Männer ärgen sich!"
         source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
-        tree = self.parser.parse_source(source.encode("utf-7"))
-        assert tree.value == "utf-7"
-        s = ast_from_node(space, tree).body[0].value
+        info = pyparse.CompileInfo("<test>", "exec")
+        tree = self.parser.parse_source(source.encode("utf-7"), info)
+        assert info.encoding == "utf-7"
+        s = ast_from_node(space, tree, info).body[0].value
         assert isinstance(s, ast.Str)
         assert space.eq_w(s.s, space.wrap(sentence))
 

Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py	Wed Jul 15 06:04:13 2009
@@ -1,4 +1,7 @@
 import py
+from pypy.interpreter.astcompiler import codegen, astbuilder
+from pypy.interpreter.pyparser import pyparse
+
 from pypy.interpreter.astcompiler import misc, pycodegen, opt
 from pypy.interpreter.pyparser.test.support import source2ast
 from pypy.interpreter.pyparser.test import expressions
@@ -6,20 +9,11 @@
 from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
 
 def compile_with_astcompiler(expr, mode, space):
-    ast = source2ast(expr, mode, space)
-    misc.set_filename('<testing>', ast)
-    ast = opt.optimize_ast_tree(space, ast)
-    if mode == 'exec':
-        Generator = pycodegen.ModuleCodeGenerator
-    elif mode == 'single':
-        Generator = pycodegen.InteractiveCodeGenerator
-    elif mode == 'eval':
-        Generator = pycodegen.ExpressionCodeGenerator
-    codegen = Generator(space, ast)
-    rcode = codegen.getCode()
-    assert isinstance(rcode, PyCode)
-    assert rcode.co_filename == '<testing>'
-    return rcode
+    p = pyparse.PythonParser(space)
+    info = pyparse.CompileInfo("<test>", mode)
+    cst = p.parse_source(expr, info)
+    ast = astbuilder.ast_from_node(space, cst, info)
+    return codegen.compile_ast(space, ast, info)
 
 
 class TestCompiler:
@@ -654,7 +648,7 @@
         try:
             self.simple_test(source, None, None)
         except IndentationError, e:
-            assert e.msg == 'expected an indented block'
+            assert e.msg == 'expected indented block'
         else:
             raise Exception("DID NOT RAISE")
 

Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py	Wed Jul 15 06:04:13 2009
@@ -11,9 +11,10 @@
         cls.parser = pyparse.PythonParser(cls.space)
 
     def mod_scope(self, source, mode="exec"):
-        tree = self.parser.parse_source(source)
-        module = astbuilder.ast_from_node(self.space, tree)
-        builder = symtable.SymtableBuilder(self.space, module)
+        info = pyparse.CompileInfo("<test>", mode)
+        tree = self.parser.parse_source(source, info)
+        module = astbuilder.ast_from_node(self.space, tree, info)
+        builder = symtable.SymtableBuilder(self.space, module, info)
         scope = builder.find_scope(module)
         assert isinstance(scope, symtable.ModuleScope)
         return scope

Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py	Wed Jul 15 06:04:13 2009
@@ -205,6 +205,7 @@
 
 ########
 
+
 class PythonAstCompiler(PyCodeCompiler):
     """Uses the stdlib's python implementation of compiler
 
@@ -215,44 +216,30 @@
     """
     def __init__(self, space, override_version=None):
 
-        from pyparser.pythonparse import make_pyparser
+        from pypy.interpreter.pyparser.pyparse import PythonParser
         PyCodeCompiler.__init__(self, space)
-        self.grammar_version = override_version or "2.5"
-        self.parser = make_pyparser(self.grammar_version)
+        self.parser = PythonParser(space)
         self.additional_rules = {}
-        if self.grammar_version >= '2.5':
-            self.futureFlags = future.futureFlags_2_5
-        else:
-            self.futureFlags = future.futureFlags_2_4
+        self.futureFlags = future.futureFlags_2_5
         self.compiler_flags = self.futureFlags.allowed_flags
 
     def compile(self, source, filename, mode, flags):
         from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
-        from pypy.interpreter import astcompiler
-        from pypy.interpreter.astcompiler.pycodegen import ModuleCodeGenerator
-        from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator
-        from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator
-        from pypy.interpreter.astcompiler.ast import Node
-        from pypy.interpreter.astcompiler import opt
-        from pyparser.astbuilder import AstBuilder
         from pypy.interpreter.pycode import PyCode
-        from pypy.interpreter.function import Function
-
+        from pypy.interpreter.pyparser.pyparse import CompileInfo
         from pypy.interpreter.pyparser.future import getFutures
         from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError
+        from pypy.interpreter.astcompiler.astbuilder import ast_from_node
+        from pypy.interpreter.astcompiler.codegen import compile_ast
 
-##         flags |= stdlib___future__.generators.compiler_flag   # always on (2.2 compat)
         space = self.space
         space.timer.start("PythonAST compile")
         try:
-            builder = AstBuilder(self.parser, self.grammar_version, space=space)
-            for rulename, buildfunc in self.additional_rules.iteritems():
-                assert isinstance(buildfunc, Function)
-                builder.user_build_rules[rulename] = buildfunc
             flags |= getFutures(self.futureFlags, source)
-            self.parser.parse_source(source, mode, builder, flags)
-            ast_tree = builder.rule_stack[-1]
-            encoding = builder.source_encoding
+            info = CompileInfo(filename, mode, flags)
+            parse_tree = self.parser.parse_source(source, info)
+            module = ast_from_node(space, parse_tree, info)
+            code = compile_ast(space, module, info)
         except IndentationError, e:
             raise OperationError(space.w_IndentationError,
                                  e.wrap_info(space, filename))
@@ -262,36 +249,9 @@
         except SyntaxError, e:
             raise OperationError(space.w_SyntaxError,
                                  e.wrap_info(space, filename))
-        ast_tree = opt.optimize_ast_tree(space, ast_tree)
-
-        if not space.is_w(self.w_compile_hook, space.w_None):
-            try:
-                w_ast_tree = space.call_function(self.w_compile_hook,
-                                                 space.wrap(ast_tree),
-                                                 space.wrap(encoding),
-                                                 space.wrap(filename))
-                ast_tree = space.interp_w(Node, w_ast_tree)
-            except OperationError:
-                self.w_compile_hook = space.w_None
-                raise
-        try:
-            astcompiler.misc.set_filename(filename, ast_tree)
-            flag_names = self.futureFlags.get_flag_names(space, flags)
-            if mode == 'exec':
-                codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names)
-            elif mode == 'single':
-                codegenerator = InteractiveCodeGenerator(space, ast_tree, flag_names)
-            else: # mode == 'eval':
-                codegenerator = ExpressionCodeGenerator(space, ast_tree, flag_names)
-            c = codegenerator.getCode()
-        except SyntaxError, e:
-            raise OperationError(space.w_SyntaxError,
-                                 e.wrap_info(space, filename))
-        except (ValueError, TypeError), e:
-            raise OperationError(space.w_SystemError, space.wrap(str(e)))
-        assert isinstance(c, PyCode)
+        assert isinstance(code, PyCode)
         space.timer.stop("PythonAST compile")
-        return c
+        return code
 
     # interface for pypy.module.recparser
     def get_parser(self):

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py	Wed Jul 15 06:04:13 2009
@@ -56,6 +56,15 @@
     return pytokenizer.match_encoding_declaration(line[i:])
 
 
+class CompileInfo(object):
+
+    def __init__(self, filename, mode="exec", flags=0):
+        self.filename = filename
+        self.mode = mode
+        self.encoding = None
+        self.flags = flags
+
+
 _targets = {
 'eval' : pygram.syms.eval_input,
 'single' : pygram.syms.single_input,
@@ -68,7 +77,7 @@
         parser.Parser.__init__(self, grammar)
         self.space = space
 
-    def parse_source(self, textsrc, mode="exec", flags=0):
+    def parse_source(self, textsrc, compile_info=None):
         """Parse a python source according to goal"""
         # Detect source encoding.
         enc = None
@@ -93,9 +102,9 @@
                         raise error.SyntaxError("Unknown encoding: %s" % enc)
                     raise
 
-        self.prepare(_targets[mode])
+        self.prepare(_targets[compile_info.mode])
         try:
-            tokens = pytokenizer.generate_tokens(textsrc, flags)
+            tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags)
             for tp, value, lineno, column, line in tokens:
                 if self.add_token(tp, value, lineno, column, line):
                     break
@@ -108,12 +117,12 @@
             else:
                 new_err = error.SyntaxError
                 msg = "invalid syntax"
-            raise new_err(msg, e.lineno, e.column, e.line)
+            raise new_err(msg, e.lineno, e.column, e.line,
+                          compile_info.filename)
         else:
             tree = self.root
         finally:
             self.root = None
         if enc is not None:
-            # Wrap the tree in an encoding_decl node for the AST builder.
-            tree = parser.Node(pygram.syms.encoding_decl, enc, [tree], 0, 0)
+            compile_info.encoding = enc
         return tree

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	Wed Jul 15 06:04:13 2009
@@ -10,27 +10,33 @@
     def setup_class(self):
         self.parser = pyparse.PythonParser(self.space)
 
+    def parse(self, source, mode="exec", info=None):
+        if info is None:
+            info = pyparse.CompileInfo("<test>", mode)
+        return self.parser.parse_source(source, info)
+
     def test_clear_state(self):
         assert self.parser.root is None
-        tree = self.parser.parse_source("name = 32")
+        tree = self.parse("name = 32")
         assert self.parser.root is None
 
     def test_encoding(self):
-        tree = self.parser.parse_source("""# coding: latin-1
+        info = pyparse.CompileInfo("<test>", "exec")
+        tree = self.parse("""# coding: latin-1
 stuff = "nothing"
-""")
-        assert tree.type == syms.encoding_decl
-        assert tree.value == "iso-8859-1"
+""", info=info)
+        assert tree.type == syms.file_input
+        assert info.encoding == "iso-8859-1"
         sentence = u"u'Die Männer ärgen sich!'"
         input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7")
-        tree = self.parser.parse_source(input)
-        assert tree.value == "utf-7"
+        tree = self.parse(input, info=info)
+        assert info.encoding == "utf-7"
         input = "# coding: not-here"
-        exc = py.test.raises(SyntaxError, self.parser.parse_source, input).value
+        exc = py.test.raises(SyntaxError, self.parse, input).value
         assert exc.msg == "Unknown encoding: not-here"
 
     def test_syntax_error(self):
-        parse = self.parser.parse_source
+        parse = self.parse
         exc = py.test.raises(SyntaxError, parse, "name another for").value
         assert exc.msg == "invalid syntax"
         assert exc.lineno == 1
@@ -44,11 +50,11 @@
             py.test.raises(SyntaxError, parse, input)
 
     def test_is(self):
-        self.parser.parse_source("x is y")
-        self.parser.parse_source("x is not y")
+        self.parse("x is y")
+        self.parse("x is not y")
 
     def test_indentation_error(self):
-        parse = self.parser.parse_source
+        parse = self.parse
         input = """
 def f():
 pass"""
@@ -65,9 +71,9 @@
         assert exc.msg == "unindent does not match any outer indentation level"
 
     def test_mode(self):
-        assert self.parser.parse_source("x = 43*54").type == syms.file_input
-        tree = self.parser.parse_source("43**54", "eval")
+        assert self.parse("x = 43*54").type == syms.file_input
+        tree = self.parse("43**54", "eval")
         assert tree.type == syms.eval_input
-        py.test.raises(SyntaxError, self.parser.parse_source, "x = 54", "eval")
-        tree = self.parser.parse_source("x = 43", "single")
+        py.test.raises(SyntaxError, self.parse, "x = 54", "eval")
+        tree = self.parse("x = 43", "single")
         assert tree.type == syms.single_input



More information about the Pypy-commit mailing list