[pypy-svn] r12759 - pypy/branch/pycompiler/module/recparser
ludal at codespeak.net
ludal at codespeak.net
Tue May 24 11:52:20 CEST 2005
Author: ludal
Date: Tue May 24 11:52:20 2005
New Revision: 12759
Modified:
pypy/branch/pycompiler/module/recparser/__init__.py
pypy/branch/pycompiler/module/recparser/compat.py
pypy/branch/pycompiler/module/recparser/pyparser.py
pypy/branch/pycompiler/module/recparser/pythonparse.py
pypy/branch/pycompiler/module/recparser/pythonutil.py
pypy/branch/pycompiler/module/recparser/syntaxtree.py
Log:
* reorganisation : move around some functions and defs
* add ast_xxx function family to produce ast tree from source
Modified: pypy/branch/pycompiler/module/recparser/__init__.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/__init__.py (original)
+++ pypy/branch/pycompiler/module/recparser/__init__.py Tue May 24 11:52:20 2005
@@ -3,10 +3,8 @@
from pypy.interpreter.mixedmodule import MixedModule
-import pythonutil
-
debug_print( "Loading grammar %s" % pythonutil.PYTHON_GRAMMAR )
-PYTHON_PARSER = pythonutil.python_grammar()
+import pythonutil
class Module(MixedModule):
"""The builtin parser module.
Modified: pypy/branch/pycompiler/module/recparser/compat.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/compat.py (original)
+++ pypy/branch/pycompiler/module/recparser/compat.py Tue May 24 11:52:20 2005
@@ -1,7 +1,7 @@
"""Compatibility layer for CPython's parser module"""
from pythonparse import parse_python_source
-from pypy.module.recparser import PYTHON_PARSER
+from pythonutil import PYTHON_PARSER
from compiler import transformer, compile as pycompile
def suite( source ):
Modified: pypy/branch/pycompiler/module/recparser/pyparser.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/pyparser.py (original)
+++ pypy/branch/pycompiler/module/recparser/pyparser.py Tue May 24 11:52:20 2005
@@ -9,7 +9,7 @@
from pypy.interpreter.pycode import PyCode
from syntaxtree import SyntaxNode
from pythonparse import parse_python_source
-from pypy.module.recparser import PYTHON_PARSER
+from pythonutil import PYTHON_PARSER
__all__ = [ "ASTType", "STType", "suite", "expr" ]
Modified: pypy/branch/pycompiler/module/recparser/pythonparse.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/pythonparse.py (original)
+++ pypy/branch/pycompiler/module/recparser/pythonparse.py Tue May 24 11:52:20 2005
@@ -1,16 +1,32 @@
#!/usr/bin/env python
-from grammar import BaseGrammarBuilder
from pythonlexer import PythonSource
from ebnfparse import parse_grammar
import sys
-import pythonutil
+import os
import symbol
+import grammar
-def parse_python_source( textsrc, gram, goal ):
+# parse the python grammar corresponding to our CPython version
+_ver = ".".join([str(i) for i in sys.version_info[:2]])
+PYTHON_GRAMMAR = os.path.join( os.path.dirname(__file__), "data", "Grammar" + _ver )
+
+def python_grammar():
+ """returns a """
+ level = grammar.DEBUG
+ grammar.DEBUG = 0
+ gram = parse_grammar( file(PYTHON_GRAMMAR) )
+ grammar.DEBUG = level
+ return gram
+
+PYTHON_PARSER = python_grammar()
+
+
+def parse_python_source( textsrc, gram, goal, builder=None ):
"""Parse a python source according to goal"""
target = gram.rules[goal]
src = PythonSource(textsrc)
- builder = BaseGrammarBuilder(debug=False, rules=gram.rules)
+ if builder is None:
+ builder = grammar.BaseGrammarBuilder(debug=False, rules=gram.rules)
result = target.match(src, builder)
# <HACK> XXX find a clean way to process encoding declarations
if src.encoding:
@@ -20,46 +36,17 @@
raise SyntaxError("at %s" % src.debug() )
return builder
-def parse_file_input(pyf, gram):
+def parse_file_input(pyf, gram, builder=None):
"""Parse a python file"""
- return parse_python_source( pyf.read(), gram, "file_input" )
+ return parse_python_source( pyf.read(), gram, "file_input", builder )
-def parse_single_input(textsrc, gram):
- """Parse a python file"""
- return parse_python_source( textsrc, gram, "single_input" )
+def parse_single_input(textsrc, gram, builder=None):
+ """Parse a python single statement"""
+ return parse_python_source( textsrc, gram, "single_input", builder )
+
+def parse_eval_input(textsrc, gram, builder=None):
+ """Parse a python expression"""
+ return parse_python_source( textsrc, gram, "eval_input", builder )
-def parse_eval_input(textsrc, gram):
- """Parse a python file"""
- return parse_python_source( textsrc, gram, "eval_input" )
-def pypy_parse(filename):
- """parse <filename> using PyPy's parser module and return nested tuples
- """
- pyf = file(filename)
- builder = parse_file_input(pyf, pythonutil.python_grammar())
- pyf.close()
- if builder.stack:
- # print builder.stack[-1]
- root_node = builder.stack[-1]
- nested_tuples = root_node.totuple()
- if hasattr(builder, '_source_encoding'):
- # XXX: maybe the parser could fix that instead ?
- return ( symbol.encoding_decl, nested_tuples, builder._source_encoding)
- else:
- return nested_tuples
- return None # XXX raise an exception instead
-
-if __name__ == "__main__":
- if len(sys.argv) < 2:
- print "python parse.py [-d N] test_file.py"
- sys.exit(1)
- if sys.argv[1] == "-d":
- debug_level = int(sys.argv[2])
- test_file = sys.argv[3]
- else:
- test_file = sys.argv[1]
- print "-"*20
- print
- print "pyparse \n", pypy_parse(test_file)
- print "parser \n", pythonutil.python_parse(test_file)
Modified: pypy/branch/pycompiler/module/recparser/pythonutil.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/pythonutil.py (original)
+++ pypy/branch/pycompiler/module/recparser/pythonutil.py Tue May 24 11:52:20 2005
@@ -1,30 +1,11 @@
-__all__ = ["python_grammar", "PYTHON_GRAMMAR" ]
+__all__ = ["python_parse", "pypy_parse","ast_single_input", "ast_file_input", "ast_eval_input" ]
-import os
-import sys
-
-_ver = ".".join([str(i) for i in sys.version_info[:2]])
-PYTHON_GRAMMAR = os.path.join( os.path.dirname(__file__), "data", "Grammar" + _ver )
-
-def python_grammar():
- """returns a """
- from ebnfparse import parse_grammar
- level = get_debug()
- set_debug( 0 )
- gram = parse_grammar( file(PYTHON_GRAMMAR) )
- set_debug( level )
- return gram
-
-def get_debug():
- """Return debug level"""
- import grammar
- return grammar.DEBUG
-
-def set_debug( level ):
- """sets debug mode to <level>"""
- import grammar
- grammar.DEBUG = level
+import grammar
+import pythonparse
+from compiler.transformer import Transformer
+from tuplebuilder import TupleBuilder
+PYTHON_PARSER = pythonparse.PYTHON_PARSER
def python_parse(filename):
"""parse <filename> using CPython's parser module and return nested tuples
@@ -33,3 +14,64 @@
import parser
tp2 = parser.suite(pyf.read())
return tp2.totuple()
+
+def pypy_parse(filename):
+ """parse <filename> using PyPy's parser module and return nested tuples
+ """
+ pyf = file(filename)
+ builder = parse_file_input(pyf, pythonutil.PYTHON_PARSER )
+ pyf.close()
+ if builder.stack:
+ # print builder.stack[-1]
+ root_node = builder.stack[-1]
+ nested_tuples = root_node.totuple()
+ if hasattr(builder, '_source_encoding'):
+ # XXX: maybe the parser could fix that instead ?
+ return ( symbol.encoding_decl, nested_tuples, builder._source_encoding)
+ else:
+ return nested_tuples
+ return None # XXX raise an exception instead
+
+
+def ast_single_input( text ):
+ builder = TupleBuilder( PYTHON_PARSER.rules )
+ pythonparse.parse_python_source( text, PYTHON_PARSER, "single_input", builder )
+ tree = builder.stack[-1]
+ trans = Transformer()
+ ast = trans.transform( tree )
+ return ast
+
+def ast_file_input( filename ):
+ pyf = file(filename,"r")
+ text = pyf.read()
+ builder = TupleBuilder( PYTHON_PARSER.rules )
+ pythonparse.parse_python_source( text, PYTHON_PARSER, "file_input", builder )
+ tree = builder.stack[-1]
+ trans = Transformer()
+ ast = trans.transform( tree )
+ return ast, tree
+
+def ast_eval_input( textsrc ):
+ builder = TupleBuilder( PYTHON_PARSER.rules )
+ pythonparse.parse_python_source( textsrc, PYTHON_PARSER, "eval_input", builder )
+ tree = builder.stack[-1]
+ trans = Transformer()
+ ast = trans.transform( tree )
+ return ast
+
+
+
+if __name__ == "__main__":
+ import sys
+ if len(sys.argv) < 2:
+ print "python parse.py [-d N] test_file.py"
+ sys.exit(1)
+ if sys.argv[1] == "-d":
+ debug_level = int(sys.argv[2])
+ test_file = sys.argv[3]
+ else:
+ test_file = sys.argv[1]
+ print "-"*20
+ print
+ print "pyparse \n", pypy_parse(test_file)
+ print "parser \n", python_parse(test_file)
Modified: pypy/branch/pycompiler/module/recparser/syntaxtree.py
==============================================================================
--- pypy/branch/pycompiler/module/recparser/syntaxtree.py (original)
+++ pypy/branch/pycompiler/module/recparser/syntaxtree.py Tue May 24 11:52:20 2005
@@ -60,7 +60,7 @@
"|" : token.VBAR,
"|=" : token.VBAREQUAL,
}
-
+NT_OFFSET = token.NT_OFFSET
SYMBOLS = {}
# copies the numerical mapping between symbol name and symbol value
# into SYMBOLS
More information about the Pypy-commit
mailing list