[pypy-svn] r23777 - in pypy/dist/pypy/interpreter/pyparser: . data test

stuart at codespeak.net stuart at codespeak.net
Tue Feb 28 20:10:33 CET 2006


Author: stuart
Date: Tue Feb 28 20:10:32 2006
New Revision: 23777

Added:
   pypy/dist/pypy/interpreter/pyparser/data/Grammar_stablecompiler
      - copied unchanged from r23764, pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a
Modified:
   pypy/dist/pypy/interpreter/pyparser/pythonparse.py
   pypy/dist/pypy/interpreter/pyparser/pythonutil.py
   pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
   pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py
Log:
(Arre, Ale, Stuart Williams)
Changed parsing tests to use stable compiler with a frozen grammar
file in order to be able change the grammar without breaking the
stable compiler used in testing.
Started adding support for tests of AST generation of new grammar elements.



Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py	Tue Feb 28 20:10:32 2006
@@ -112,6 +112,8 @@
     """returns the python grammar corresponding to our CPython version"""
     if version == "native":
         _ver = PYTHON_VERSION
+    elif version == "stable":
+        _ver = "_stablecompiler"
     elif version in ("2.3","2.4","2.5a"):
         _ver = version
     return os.path.join( os.path.dirname(__file__), "data", "Grammar" + _ver ), _ver

Modified: pypy/dist/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonutil.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonutil.py	Tue Feb 28 20:10:32 2006
@@ -50,7 +50,8 @@
     pyf.close()
     return pypy_parse(source, 'exec', lineno)
 
-def internal_pypy_parse(source, mode='exec', lineno=False, flags=0, space=None):
+def internal_pypy_parse(source, mode='exec', lineno=False, flags=0, space=None,
+                        parser = PYTHON_PARSER):
     """This function has no other role than testing the parser's annotation
 
     annotateme() is basically the same code that pypy_parse(), but with the
@@ -60,11 +61,11 @@
        tuples (StackElement is only a wrapper class around these tuples)
 
     """
-    builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
+    builder = TupleBuilder(parser.rules, lineno=False)
     if space is not None:
         builder.space = space
     target_rule = TARGET_DICT[mode]
-    PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
+    parser.parse_source(source, target_rule, builder, flags)
     stack_element = builder.stack[-1]
     return (builder.source_encoding, stack_element)
 
@@ -77,7 +78,7 @@
     else:
         return nested_tuples
 
-def pypy_parse(source, mode='exec', lineno=False, flags=0):
+def pypy_parse(source, mode='exec', lineno=False, flags=0, parser = PYTHON_PARSER):
     """
     NOT_RPYTHON !
     parse <source> using PyPy's parser module and return
@@ -90,13 +91,14 @@
      - The encoding string or None if there were no encoding statement
     nested tuples
     """
-    source_encoding, stack_element = internal_pypy_parse(source, mode, lineno=lineno, flags=lineno)
+    source_encoding, stack_element = internal_pypy_parse(source, mode, lineno=lineno,
+                                                         flags=lineno, parser = parser)
     # convert the stack element into nested tuples (caution, the annotator
     # can't follow this call)
     return parse_result_to_nested_tuples((source_encoding, stack_element), lineno=lineno)
 
 ## convenience functions for computing AST objects using recparser
-def ast_from_input(input, mode, transformer):
+def ast_from_input(input, mode, transformer, parser = PYTHON_PARSER):
     """converts a source input into an AST
 
      - input : the source to be converted
@@ -107,7 +109,7 @@
           here to explicitly import compiler or stablecompiler or
           etc. This is to be fixed in a clean way
     """
-    tuples = pypy_parse(input, mode, True)
+    tuples = pypy_parse(input, mode, True, parser)
     ast = transformer.compile_node(tuples)
     return ast
 

Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py	Tue Feb 28 20:10:32 2006
@@ -1,6 +1,6 @@
 import os
 
-from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser import pythonparse
 from pypy.interpreter.pyparser.astbuilder import AstBuilder
 from pypy.interpreter.pyparser.pythonutil import ast_from_input
 from pypy.interpreter.stablecompiler.transformer import Transformer
@@ -704,12 +704,16 @@
 def ast_parse_expr(expr, target='single'):
     target = TARGET_DICT[target]
     builder = AstBuilder(space=FakeSpace())
-    PYTHON_PARSER.parse_source(expr, target, builder)
+    pythonparse.PYTHON_PARSER.parse_source(expr, target, builder)
     return builder
 
+# Create parser from Grammar_stable, not current grammar.
+stable_grammar, _ = pythonparse.get_grammar_file("stable")
+stable_parser = pythonparse.python_grammar(stable_grammar)
+
 def tuple_parse_expr(expr, target='single'):
     t = Transformer("dummyfile")
-    return ast_from_input(expr, target, t)
+    return ast_from_input(expr, target, t, stable_parser)
 
 def check_expression(expr, target='single'):
     r1 = ast_parse_expr(expr, target)
@@ -786,6 +790,11 @@
     for snippet_name in SNIPPETS:
         filepath = os.path.join(os.path.dirname(__file__), 'samples', snippet_name)
         source = file(filepath).read()
+        # To avoid using the stable compiler we pull an explicit AST out of the snippet
+        if source.startswith('# EXPECT:'):
+            firstline,_ = source.split('\n', 1)
+            firstline = firstline[len('# EXPECT:'):].strip()
+            EXPECTED[source] = firstline
         yield check_expression, source, 'exec'
 
 def test_libstuff():

Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py	Tue Feb 28 20:10:32 2006
@@ -1,5 +1,5 @@
 import os
-from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser import pythonparse
 from pypy.interpreter.pyparser.astbuilder import AstBuilder
 from pypy.interpreter.pyparser.tuplebuilder import TupleBuilder
 from pypy.interpreter.pycode import PyCode
@@ -63,7 +63,7 @@
 def ast_parse_expr(expr, target='single', space=FakeSpace()):
     target = TARGET_DICT[target]
     builder = AstBuilder(space=space)
-    PYTHON_PARSER.parse_source(expr, target, builder)
+    pythonparse.PYTHON_PARSER.parse_source(expr, target, builder)
     return builder.rule_stack[-1]
 
 
@@ -80,10 +80,15 @@
     rcode = codegen.getCode()
     return rcode
 
+
+# Create parser from Grammar_stable, not current grammar.
+stable_grammar, _ = pythonparse.get_grammar_file("stable")
+stable_parser = pythonparse.python_grammar(stable_grammar)
+
 def compile_with_testcompiler(expr, target='exec', space=FakeSpace()):
     target2 = TARGET_DICT['exec'] # xxx exec: single not really tested
     builder = TupleBuilder()
-    PYTHON_PARSER.parse_source(expr, target2, builder)
+    stable_parser.parse_source(expr, target2, builder)
     tuples =  builder.stack[-1].as_tuple(True)
     from pypy.interpreter.stablecompiler import transformer, pycodegen, misc
     ast = transformer.Transformer('<?>').compile_node(tuples)



More information about the Pypy-commit mailing list