[pypy-svn] r14137 - in pypy/dist/pypy/interpreter: . pyparser pyparser/test

adim at codespeak.net adim at codespeak.net
Sun Jul 3 14:50:00 CEST 2005


Author: adim
Date: Sun Jul  3 14:49:56 2005
New Revision: 14137

Modified:
   pypy/dist/pypy/interpreter/pycompiler.py
   pypy/dist/pypy/interpreter/pyparser/pythonparse.py
   pypy/dist/pypy/interpreter/pyparser/pythonutil.py
   pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
Log:
- various tidy machinery
- added some small tests for 'eval' and 'exec' modes
- removed pycompile() function from pycompiler.py since this
  functionnality is now provided by pyparser.pythonutil module



Modified: pypy/dist/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/pycompiler.py	(original)
+++ pypy/dist/pypy/interpreter/pycompiler.py	Sun Jul  3 14:49:56 2005
@@ -169,30 +169,13 @@
 
 
 ########
-import symbol
-from compiler.transformer import Transformer
+# from compiler.transformer import Transformer
 from compiler.pycodegen import ModuleCodeGenerator
 from compiler.pycodegen import InteractiveCodeGenerator
 from compiler.pycodegen import ExpressionCodeGenerator
 from pyparser.pythonparse import parse_python_source, PYTHON_PARSER
 from pyparser.tuplebuilder import TupleBuilder
-
-def _parse_source(source, mode):
-    strings = [line+'\n' for line in source.split('\n')]
-    builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
-    if mode == 'exec':
-        target = 'file_input'
-    elif mode == 'single':
-        target = 'single_input'
-    else: # target == 'eval':
-        target = 'eval_input'
-    parse_python_source(strings, PYTHON_PARSER, target, builder)
-    # Note: The annotator can't follow the as_tuple() method call
-    nested_tuples = builder.stack[-1].as_tuple()
-    if builder.source_encoding is not None:
-        return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
-    else:
-        return nested_tuples
+from pyparser.pythonutil import ast_from_input
 
 class PythonCompiler(CPythonCompiler):
     """Uses the stdlib's python implementation of compiler"""
@@ -201,9 +184,7 @@
         flags |= __future__.generators.compiler_flag   # always on (2.2 compat)
         space = self.space
         try:
-            transformer = Transformer()
-            tuples = _parse_source(source, mode)
-            tree = transformer.compile_node(tuples)
+            ast = ast_from_input(source, mode)
             compiler.misc.set_filename(filename, tree)
             if mode == 'exec':
                 codegenerator = ModuleCodeGenerator(tree)

Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py	Sun Jul  3 14:49:56 2005
@@ -42,9 +42,10 @@
     builder.source_encoding = src.encoding
     # </HACK>
     if not result:
-        #raise SyntaxError("...")
+        # raising a SyntaxError here is not annotable, and it can
+        # probably be handled in an other way
+        # raise SyntaxError("at %s" % src.debug() )
         return None
-    # raise SyntaxError("at %s" % src.debug() )
     return builder
 
 def parse_file_input(pyf, gram, builder=None):

Modified: pypy/dist/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonutil.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonutil.py	Sun Jul  3 14:49:56 2005
@@ -1,22 +1,41 @@
 __all__ = ["python_parse", "pypy_parse", "ast_single_input", "ast_file_input",
            "ast_eval_input" ]
 
-import pythonparse
 from compiler.transformer import Transformer
+import parser
+import symbol
+
+import pythonparse
 from tuplebuilder import TupleBuilder
 
 PYTHON_PARSER = pythonparse.PYTHON_PARSER
+TARGET_DICT = {
+    'exec'   : "file_input",
+    'eval'   : "eval_input",
+    'single' : "single_input",
+    }
 
-def python_parse(filename):
+## convenience functions around CPython's parser functions
+def python_parsefile(filename):
     """parse <filename> using CPython's parser module and return nested tuples
     """
     pyf = file(filename)
-    import parser
-    tp2 = parser.suite(pyf.read())
-    return tp2.totuple()
+    source = pyf.read()
+    pyf.close()
+    return python_parse(source)
 
-import symbol
-def pypy_parse(filename):
+def python_parse(source, mode='exec'):
+    """parse python source using CPython's parser module and return
+    nested tuples
+    """
+    if mode == 'exec':
+        tp = parser.suite(source)
+    else:
+        tp = parser.expr(source)
+    return tp.totuple()
+
+## convenience functions around recparser functions
+def pypy_parsefile(filename):
     """parse <filename> using PyPy's parser module and return
     a tuple of three elements :
      - The encoding declaration symbol or None if there were no encoding
@@ -28,20 +47,63 @@
     nested tuples
     """
     pyf = file(filename)
-    text = pyf.read()
+    source = pyf.read()
     pyf.close()
+    return pypy_parse(source)
+
+def pypy_parse(source, mode='exec'):
+    """parse <source> using PyPy's parser module and return
+    a tuple of three elements :
+     - The encoding declaration symbol or None if there were no encoding
+       statement
+     - The TupleBuilder's stack top element (instance of
+       tuplebuilder.StackElement which is a wrapper of some nested tuples
+       like those returned by the CPython's parser)
+     - The encoding string or None if there were no encoding statement
+    nested tuples
+    """
     builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
-    # make the annotator life easier
-    strings = [line + '\n' for line in text.split('\n')]
+    # make the annotator life easier (don't use str.splitlines())
+    strings = [line + '\n' for line in source.split('\n')]
+    # finalize the last line 
+    if not source.endswith('\n'):
+        last_line = strings[-1]
+        strings[-1] = last_line[:-1]
+    else:
+        strings.pop()
+    target_rule = TARGET_DICT[mode]
     pythonparse.parse_python_source(strings, PYTHON_PARSER,
-                                    'file_input', builder)
-    nested_tuples = builder.stack[-1]
+                                    target_rule, builder)
+    # stack_element is a tuplerbuilder.StackElement's instance
+    stack_element = builder.stack[-1]
+    # convert the stack element into nested tuples
+    # XXX : the annotator can't follow this call
+    nested_tuples = stack_element.as_tuple()
     if builder.source_encoding is not None:
         return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
     else:
-        return (None, nested_tuples, None)
+        return nested_tuples
 
+## convenience functions for computing AST objects based on recparser
+def ast_from_input(input, mode):
+    tuples = pypy_parse(input, mode)
+    transformer = Transformer()
+    ast = transformer.transform(tuples)
+    return ast
+
+## TARGET FOR ANNOTATORS #############################################
 def annotateme(strings):
+    """This function has no other role than testing the parser's annotation
+
+    annotateme() is basically the same code that pypy_parse(), but with the
+    following differences :
+     - directly take a list of strings rather than a filename in input
+       in order to avoid using file() (which is faked for now)
+       
+     - returns a tuplebuilder.StackElement instead of the *real* nested
+       tuples (StackElement is only a wrapper class around these tuples)
+
+    """
     builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
     pythonparse.parse_python_source(strings, PYTHON_PARSER, 'file_input', builder)
     nested_tuples = builder.stack[-1]
@@ -50,37 +112,6 @@
     else:
         return (None, nested_tuples, None)
 
-def ast_single_input( text ):
-    builder = TupleBuilder( PYTHON_PARSER.rules )
-    pythonparse.parse_python_source( text, PYTHON_PARSER, "single_input", builder )
-    tree = builder.stack[-1]
-    trans = Transformer()
-    ast = trans.transform( tree )
-    return ast
-
-def ast_file_input( filename ):
-    pyf = file(filename,"r")
-    text = pyf.read()
-    return ast_srcfile_input( text, filename )
-
-def ast_srcfile_input( srctext, filename ):
-    # TODO do something with the filename
-    builder = TupleBuilder( PYTHON_PARSER.rules )
-    pythonparse.parse_python_source( srctext, PYTHON_PARSER, "file_input", builder )
-    tree = builder.stack[-1]
-    trans = Transformer()
-    ast = trans.transform( tree )
-    return ast
-
-def ast_eval_input( textsrc ):
-    builder = TupleBuilder( PYTHON_PARSER.rules )
-    pythonparse.parse_python_source( textsrc, PYTHON_PARSER, "eval_input", builder )
-    tree = builder.stack[-1]
-    trans = Transformer()
-    ast = trans.transform( tree )
-    return ast
-
-
 
 if __name__ == "__main__":
     import sys
@@ -94,5 +125,5 @@
         test_file = sys.argv[1]
     print "-"*20
     print
-    print "pyparse \n", pypy_parse(test_file)
-    print "parser  \n", python_parse(test_file)
+    print "pyparse \n", pypy_parsefile(test_file)
+    print "parser  \n", python_parsefile(test_file)

Modified: pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/test/test_samples.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_samples.py	Sun Jul  3 14:49:56 2005
@@ -1,7 +1,7 @@
 """test module for CPython / PyPy nested tuples comparison"""
 import os, os.path as osp
-from pypy.interpreter.pyparser.pythonutil import python_parse, pypy_parse
-# from pypy.module.recparser.pythonparse import pypy_parse
+from pypy.interpreter.pyparser.pythonutil import python_parsefile, \
+    pypy_parsefile, python_parse, pypy_parse
 from pprint import pprint
 from pypy.interpreter.pyparser import grammar
 grammar.DEBUG = False
@@ -54,27 +54,46 @@
                 continue
             abspath = osp.join(samples_dir, fname)
             yield check_parse, abspath
-        
-def check_parse(filepath):
-    # pypy_tuples = pypy_parse(filepath)
-    encoding_decl, stack_element, encoding = pypy_parse(filepath)
-    nested_tuples = stack_element.as_tuple()
-    if encoding is None:
-        pypy_tuples = nested_tuples
-    else:
-        pypy_tuples = (encoding_decl, nested_tuples, encoding)
-    python_tuples = python_parse(filepath)
+
+def _check_tuples_equality(pypy_tuples, python_tuples, testname):
+    """XXX FIXME: refactor with assert_tuples_equal()"""
     try:
         assert_tuples_equal(pypy_tuples, python_tuples)
     except AssertionError, e:
         error_path = e.args[-1]
         print "ERROR PATH =", error_path
-        print "="*80
-        print file(filepath).read()
-        print "="*80
         print "-"*10, "PyPy parse results", "-"*10
         print ''.join(print_sym_tuple(pypy_tuples, names=True, trace=error_path))
         print "-"*10, "CPython parse results", "-"*10
         print ''.join(print_sym_tuple(python_tuples, names=True, trace=error_path))
-        assert False, filepath
-    
+        assert False, testname
+
+def check_parse(filepath):
+    pypy_tuples = pypy_parsefile(filepath)
+    python_tuples = python_parsefile(filepath)
+    _check_tuples_equality(pypy_tuples, python_tuples, filepath)
+
+
+def check_parse_input(snippet, mode):
+    pypy_tuples = pypy_parse(snippet, mode)
+    python_tuples = python_parse(snippet, mode)
+    _check_tuples_equality(pypy_tuples, python_tuples, snippet)
+
+def test_eval_inputs():
+    snippets = [
+        '6*7',
+        'a+b*c/d',
+        'True and False',
+        ]
+    for snippet in snippets:
+        yield check_parse_input, snippet, 'eval'
+
+def test_exec_inputs():
+    snippets = [
+        # '\t # hello\n ',
+        'print 6*7', 'if 1:\n  x\n',
+        # 'if 1:\n  x', 'x = (', 'x = (\n', # 'x = (\n\n',
+        ]
+    for snippet in snippets:
+        print "snippet =", repr(snippet)
+        yield check_parse_input, snippet, 'exec'



More information about the Pypy-commit mailing list