[pypy-svn] r66223 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler astcompiler/test pyparser pyparser/test
benjamin at codespeak.net
benjamin at codespeak.net
Wed Jul 15 06:04:16 CEST 2009
Author: benjamin
Date: Wed Jul 15 06:04:13 2009
New Revision: 66223
Modified:
pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py
pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py
pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py
pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py
pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
hook the new compiler up to the interpreter
also provide a nice way, CompileInfo, to pass information like filename around
Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 06:04:13 2009
@@ -5,8 +5,8 @@
from pypy.interpreter.pyparser import parsestring
-def ast_from_node(space, n):
- return ASTBuilder(space, n).build_ast()
+def ast_from_node(space, n, compile_info):
+ return ASTBuilder(space, n, compile_info).build_ast()
augassign_operator_map = {
@@ -41,13 +41,9 @@
class ASTBuilder(object):
- def __init__(self, space, n):
+ def __init__(self, space, n, compile_info):
self.space = space
- if n.type == syms.encoding_decl:
- self.encoding = n.value
- n = n.children[0]
- else:
- self.encoding = None
+ self.compile_info = compile_info
self.root_node = n
def build_ast(self):
@@ -103,7 +99,7 @@
raise AssertionError("non-statement node")
def error(self, msg, n):
- raise SyntaxError(msg, n.lineno, n.column)
+ raise SyntaxError(msg, n.lineno, n.column, self.compile_info.filename)
def check_forbidden_name(self, name, node):
if name == "None":
@@ -1077,7 +1073,8 @@
first_child.lineno, first_child.column)
elif first_child_type == tokens.STRING:
space = self.space
- sub_strings_w = [parsestring.parsestr(space, self.encoding, s.value)
+ encoding = self.compile_info.encoding
+ sub_strings_w = [parsestring.parsestr(space, encoding, s.value)
for s in atom_node.children]
if len(sub_strings_w) > 1:
w_sub_strings = space.newlist(sub_strings_w)
Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 06:04:13 2009
@@ -253,7 +253,7 @@
class SymtableBuilder(ast.GenericASTVisitor):
- def __init__(self, space, module):
+ def __init__(self, space, module, compile_info):
self.space = space
self.module = module
self.scopes = {}
@@ -263,8 +263,12 @@
top = ModuleScope(module)
self.globs = top.roles
self.push_scope(top)
- module.walkabout(self)
- top.finalize(None, {}, {})
+ try:
+ module.walkabout(self)
+ top.finalize(None, {}, {})
+ except SyntaxError, e:
+ e.filename = compile_info.filename
+ raise
self.pop_scope()
assert not self.stack
Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 06:04:13 2009
@@ -16,8 +16,9 @@
cls.parser = pyparse.PythonParser(cls.space)
def get_ast(self, source, p_mode="exec"):
- tree = self.parser.parse_source(source, p_mode)
- ast_node = ast_from_node(self.space, tree)
+ info = pyparse.CompileInfo("<test>", p_mode)
+ tree = self.parser.parse_source(source, info)
+ ast_node = ast_from_node(self.space, tree, info)
return ast_node
def get_first_expr(self, source):
@@ -1019,9 +1020,10 @@
assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
sentence = u"Die Männer ärgen sich!"
source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
- tree = self.parser.parse_source(source.encode("utf-7"))
- assert tree.value == "utf-7"
- s = ast_from_node(space, tree).body[0].value
+ info = pyparse.CompileInfo("<test>", "exec")
+ tree = self.parser.parse_source(source.encode("utf-7"), info)
+ assert info.encoding == "utf-7"
+ s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap(sentence))
Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Wed Jul 15 06:04:13 2009
@@ -1,4 +1,7 @@
import py
+from pypy.interpreter.astcompiler import codegen, astbuilder
+from pypy.interpreter.pyparser import pyparse
+
from pypy.interpreter.astcompiler import misc, pycodegen, opt
from pypy.interpreter.pyparser.test.support import source2ast
from pypy.interpreter.pyparser.test import expressions
@@ -6,20 +9,11 @@
from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
def compile_with_astcompiler(expr, mode, space):
- ast = source2ast(expr, mode, space)
- misc.set_filename('<testing>', ast)
- ast = opt.optimize_ast_tree(space, ast)
- if mode == 'exec':
- Generator = pycodegen.ModuleCodeGenerator
- elif mode == 'single':
- Generator = pycodegen.InteractiveCodeGenerator
- elif mode == 'eval':
- Generator = pycodegen.ExpressionCodeGenerator
- codegen = Generator(space, ast)
- rcode = codegen.getCode()
- assert isinstance(rcode, PyCode)
- assert rcode.co_filename == '<testing>'
- return rcode
+ p = pyparse.PythonParser(space)
+ info = pyparse.CompileInfo("<test>", mode)
+ cst = p.parse_source(expr, info)
+ ast = astbuilder.ast_from_node(space, cst, info)
+ return codegen.compile_ast(space, ast, info)
class TestCompiler:
@@ -654,7 +648,7 @@
try:
self.simple_test(source, None, None)
except IndentationError, e:
- assert e.msg == 'expected an indented block'
+ assert e.msg == 'expected indented block'
else:
raise Exception("DID NOT RAISE")
Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 06:04:13 2009
@@ -11,9 +11,10 @@
cls.parser = pyparse.PythonParser(cls.space)
def mod_scope(self, source, mode="exec"):
- tree = self.parser.parse_source(source)
- module = astbuilder.ast_from_node(self.space, tree)
- builder = symtable.SymtableBuilder(self.space, module)
+ info = pyparse.CompileInfo("<test>", mode)
+ tree = self.parser.parse_source(source, info)
+ module = astbuilder.ast_from_node(self.space, tree, info)
+ builder = symtable.SymtableBuilder(self.space, module, info)
scope = builder.find_scope(module)
assert isinstance(scope, symtable.ModuleScope)
return scope
Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Wed Jul 15 06:04:13 2009
@@ -205,6 +205,7 @@
########
+
class PythonAstCompiler(PyCodeCompiler):
"""Uses the stdlib's python implementation of compiler
@@ -215,44 +216,30 @@
"""
def __init__(self, space, override_version=None):
- from pyparser.pythonparse import make_pyparser
+ from pypy.interpreter.pyparser.pyparse import PythonParser
PyCodeCompiler.__init__(self, space)
- self.grammar_version = override_version or "2.5"
- self.parser = make_pyparser(self.grammar_version)
+ self.parser = PythonParser(space)
self.additional_rules = {}
- if self.grammar_version >= '2.5':
- self.futureFlags = future.futureFlags_2_5
- else:
- self.futureFlags = future.futureFlags_2_4
+ self.futureFlags = future.futureFlags_2_5
self.compiler_flags = self.futureFlags.allowed_flags
def compile(self, source, filename, mode, flags):
from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
- from pypy.interpreter import astcompiler
- from pypy.interpreter.astcompiler.pycodegen import ModuleCodeGenerator
- from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator
- from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator
- from pypy.interpreter.astcompiler.ast import Node
- from pypy.interpreter.astcompiler import opt
- from pyparser.astbuilder import AstBuilder
from pypy.interpreter.pycode import PyCode
- from pypy.interpreter.function import Function
-
+ from pypy.interpreter.pyparser.pyparse import CompileInfo
from pypy.interpreter.pyparser.future import getFutures
from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError
+ from pypy.interpreter.astcompiler.astbuilder import ast_from_node
+ from pypy.interpreter.astcompiler.codegen import compile_ast
-## flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat)
space = self.space
space.timer.start("PythonAST compile")
try:
- builder = AstBuilder(self.parser, self.grammar_version, space=space)
- for rulename, buildfunc in self.additional_rules.iteritems():
- assert isinstance(buildfunc, Function)
- builder.user_build_rules[rulename] = buildfunc
flags |= getFutures(self.futureFlags, source)
- self.parser.parse_source(source, mode, builder, flags)
- ast_tree = builder.rule_stack[-1]
- encoding = builder.source_encoding
+ info = CompileInfo(filename, mode, flags)
+ parse_tree = self.parser.parse_source(source, info)
+ module = ast_from_node(space, parse_tree, info)
+ code = compile_ast(space, module, info)
except IndentationError, e:
raise OperationError(space.w_IndentationError,
e.wrap_info(space, filename))
@@ -262,36 +249,9 @@
except SyntaxError, e:
raise OperationError(space.w_SyntaxError,
e.wrap_info(space, filename))
- ast_tree = opt.optimize_ast_tree(space, ast_tree)
-
- if not space.is_w(self.w_compile_hook, space.w_None):
- try:
- w_ast_tree = space.call_function(self.w_compile_hook,
- space.wrap(ast_tree),
- space.wrap(encoding),
- space.wrap(filename))
- ast_tree = space.interp_w(Node, w_ast_tree)
- except OperationError:
- self.w_compile_hook = space.w_None
- raise
- try:
- astcompiler.misc.set_filename(filename, ast_tree)
- flag_names = self.futureFlags.get_flag_names(space, flags)
- if mode == 'exec':
- codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names)
- elif mode == 'single':
- codegenerator = InteractiveCodeGenerator(space, ast_tree, flag_names)
- else: # mode == 'eval':
- codegenerator = ExpressionCodeGenerator(space, ast_tree, flag_names)
- c = codegenerator.getCode()
- except SyntaxError, e:
- raise OperationError(space.w_SyntaxError,
- e.wrap_info(space, filename))
- except (ValueError, TypeError), e:
- raise OperationError(space.w_SystemError, space.wrap(str(e)))
- assert isinstance(c, PyCode)
+ assert isinstance(code, PyCode)
space.timer.stop("PythonAST compile")
- return c
+ return code
# interface for pypy.module.recparser
def get_parser(self):
Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 06:04:13 2009
@@ -56,6 +56,15 @@
return pytokenizer.match_encoding_declaration(line[i:])
+class CompileInfo(object):
+
+ def __init__(self, filename, mode="exec", flags=0):
+ self.filename = filename
+ self.mode = mode
+ self.encoding = None
+ self.flags = flags
+
+
_targets = {
'eval' : pygram.syms.eval_input,
'single' : pygram.syms.single_input,
@@ -68,7 +77,7 @@
parser.Parser.__init__(self, grammar)
self.space = space
- def parse_source(self, textsrc, mode="exec", flags=0):
+ def parse_source(self, textsrc, compile_info=None):
"""Parse a python source according to goal"""
# Detect source encoding.
enc = None
@@ -93,9 +102,9 @@
raise error.SyntaxError("Unknown encoding: %s" % enc)
raise
- self.prepare(_targets[mode])
+ self.prepare(_targets[compile_info.mode])
try:
- tokens = pytokenizer.generate_tokens(textsrc, flags)
+ tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags)
for tp, value, lineno, column, line in tokens:
if self.add_token(tp, value, lineno, column, line):
break
@@ -108,12 +117,12 @@
else:
new_err = error.SyntaxError
msg = "invalid syntax"
- raise new_err(msg, e.lineno, e.column, e.line)
+ raise new_err(msg, e.lineno, e.column, e.line,
+ compile_info.filename)
else:
tree = self.root
finally:
self.root = None
if enc is not None:
- # Wrap the tree in an encoding_decl node for the AST builder.
- tree = parser.Node(pygram.syms.encoding_decl, enc, [tree], 0, 0)
+ compile_info.encoding = enc
return tree
Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 06:04:13 2009
@@ -10,27 +10,33 @@
def setup_class(self):
self.parser = pyparse.PythonParser(self.space)
+ def parse(self, source, mode="exec", info=None):
+ if info is None:
+ info = pyparse.CompileInfo("<test>", mode)
+ return self.parser.parse_source(source, info)
+
def test_clear_state(self):
assert self.parser.root is None
- tree = self.parser.parse_source("name = 32")
+ tree = self.parse("name = 32")
assert self.parser.root is None
def test_encoding(self):
- tree = self.parser.parse_source("""# coding: latin-1
+ info = pyparse.CompileInfo("<test>", "exec")
+ tree = self.parse("""# coding: latin-1
stuff = "nothing"
-""")
- assert tree.type == syms.encoding_decl
- assert tree.value == "iso-8859-1"
+""", info=info)
+ assert tree.type == syms.file_input
+ assert info.encoding == "iso-8859-1"
sentence = u"u'Die Männer ärgen sich!'"
input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7")
- tree = self.parser.parse_source(input)
- assert tree.value == "utf-7"
+ tree = self.parse(input, info=info)
+ assert info.encoding == "utf-7"
input = "# coding: not-here"
- exc = py.test.raises(SyntaxError, self.parser.parse_source, input).value
+ exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "Unknown encoding: not-here"
def test_syntax_error(self):
- parse = self.parser.parse_source
+ parse = self.parse
exc = py.test.raises(SyntaxError, parse, "name another for").value
assert exc.msg == "invalid syntax"
assert exc.lineno == 1
@@ -44,11 +50,11 @@
py.test.raises(SyntaxError, parse, input)
def test_is(self):
- self.parser.parse_source("x is y")
- self.parser.parse_source("x is not y")
+ self.parse("x is y")
+ self.parse("x is not y")
def test_indentation_error(self):
- parse = self.parser.parse_source
+ parse = self.parse
input = """
def f():
pass"""
@@ -65,9 +71,9 @@
assert exc.msg == "unindent does not match any outer indentation level"
def test_mode(self):
- assert self.parser.parse_source("x = 43*54").type == syms.file_input
- tree = self.parser.parse_source("43**54", "eval")
+ assert self.parse("x = 43*54").type == syms.file_input
+ tree = self.parse("43**54", "eval")
assert tree.type == syms.eval_input
- py.test.raises(SyntaxError, self.parser.parse_source, "x = 54", "eval")
- tree = self.parser.parse_source("x = 43", "single")
+ py.test.raises(SyntaxError, self.parse, "x = 54", "eval")
+ tree = self.parse("x = 43", "single")
assert tree.type == syms.single_input
More information about the Pypy-commit
mailing list