[pypy-svn] r22540 - in pypy/branch/ast-experiments/pypy/interpreter/pyparser: . test

ludal at codespeak.net ludal at codespeak.net
Mon Jan 23 18:14:24 CET 2006


Author: ludal
Date: Mon Jan 23 18:14:22 2006
New Revision: 22540

Modified:
   pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py
   pypy/branch/ast-experiments/pypy/interpreter/pyparser/grammar.py
   pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py
   pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py
Log:
(ludal,adim)

debugging and first attempt to replace old with new grammar parser


Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py	(original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/ebnfparse.py	Mon Jan 23 18:14:22 2006
@@ -124,7 +124,7 @@
 
 def ebnf_handle_alternative(self, node):
     items = [node.nodes[0].visit(self)]
-    items += node.nodes[1].visit(self)        
+    items += node.nodes[1].visit(self)
     if len(items) == 1 and not items[0].is_root():
 	return items[0]
     alt = Alternative(self.new_symbol(), items)
@@ -203,7 +203,7 @@
 
 def handle_unknown( self, node ):
     raise RuntimeError("Unknown Visitor for %r" % node.name)
-    
+
 
 
 class EBNFBuilder(AbstractBuilder):
@@ -221,6 +221,8 @@
         self.curseqcount = 0
         self.current_subrule = 0
         self.current_rule = -1
+        self.all_rules = []
+        self.tokens = {}
 
     def new_symbol(self):
         current_rule_name = self.symbols.sym_name.get(self.current_rule,"x")
@@ -229,6 +231,35 @@
         symval = self.symbols.add_anon_symbol( rule_name )
         return symval
 
+    def new_rule(self, rule):
+        self.all_rules.append(rule)
+        return rule
+
+    def resolve_rules(self):
+        """Remove GrammarProxy objects"""
+        to_be_deleted = {}
+        for rule in self.all_rules:
+            for i, arg in enumerate(rule.args):
+                if isinstance(arg, GrammarProxy):
+                    real_rule = self.root_rules[arg.codename]
+                    if isinstance(real_rule, GrammarProxy):
+                        # If we still have a GrammarProxy associated to this codename
+                        # this means we have encountered a terminal symbol
+                        to_be_deleted[ arg.codename ] = True
+                        rule.args[i] = self.get_token( arg.codename )
+                        #print arg, "-> Token(",arg.rule_name,")" 
+                    else:
+                        #print arg, "->", real_rule
+                        rule.args[i] = real_rule
+        for codename in to_be_deleted.keys():
+            del self.root_rules[codename]
+
+    def get_token(self, codename ):
+        if codename in self.tokens:
+            return self.tokens[codename]
+        token = self.tokens[codename] = Token(codename)
+        return token
+
     def get_symbolcode(self, name ):
         codename = self.symbols.sym_values.get( name, -1 )
         if codename == -1:
@@ -236,10 +267,13 @@
         return codename
 
     def get_rule( self, name ):
+        tokencode = pytoken.tok_values.get( name, -1 )
+        if tokencode>=0:
+            return self.get_token( tokencode )
         codename = self.get_symbolcode( name )
         if codename in self.root_rules:
             return self.root_rules[codename]
-        proxy = GrammarProxy( codename )
+        proxy = GrammarProxy( name, codename )
         self.root_rules[codename] = proxy
         return proxy
 
@@ -250,11 +284,11 @@
     def restore(self, ctx):
         """Accept an opaque context object"""
         assert False, "Not supported"
-    
+
     def alternative(self, rule, source):
 #        print " alternative", rule.display(level=0,symbols=ebnfgrammar.sym_map)
         return True
-    
+
     def pop_rules( self, count ):
         offset = len(self.rule_stack)-count
         assert offset>=0
@@ -272,7 +306,7 @@
                 self.curaltcount += 1
                 return True
             rules = self.pop_rules(self.curseqcount)
-            new_rule = Sequence( self.new_symbol(), rules )
+            new_rule = self.new_rule(Sequence( self.new_symbol(), rules ))
             self.rule_stack.append( new_rule )
             self.curseqcount = 0
             self.curaltcount += 1
@@ -282,7 +316,7 @@
                 self.curaltcount = 0
                 return True
             rules = self.pop_rules(self.curaltcount)
-            new_rule = Alternative( self.new_symbol(), rules )
+            new_rule = self.new_rule(Alternative( self.new_symbol(), rules ))
             self.rule_stack.append( new_rule )
             self.curaltcount = 0
         elif _rule == ebnfgrammar.group:
@@ -314,11 +348,11 @@
             self.curseqcount += 1
         elif name == ebnfgrammar.TOK_STAR:
             top = self.rule_stack[-1]
-            rule = KleeneStar( self.new_symbol(), _min=0, rule=top)
+            rule = self.new_rule(KleeneStar( self.new_symbol(), _min=0, rule=top))
             self.rule_stack[-1] = rule
         elif name == ebnfgrammar.TOK_ADD:
             top = self.rule_stack[-1]
-            rule = KleeneStar( self.new_symbol(), _min=1, rule=top)
+            rule = self.new_rule(KleeneStar( self.new_symbol(), _min=1, rule=top))
             self.rule_stack[-1] = rule
         elif name == ebnfgrammar.TOK_BAR:
             assert self.curseqcount == 0
@@ -424,7 +458,6 @@
     node.visit(vis)
     return vis
 
-
 def parse_grammar_text(txt):
     """parses a grammar input
 
@@ -441,7 +474,7 @@
     return None
 
 def main_build():
-    from pprint import pprint    
+    from pprint import pprint
     grambuild = parse_grammar(file('data/Grammar2.3'))
     for i,r in enumerate(grambuild.items):
         print "%  3d : %s" % (i, r)

Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/grammar.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/grammar.py	(original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/grammar.py	Mon Jan 23 18:14:22 2006
@@ -215,7 +215,7 @@
     """Base parser class"""
 
     symbols = {} # dirty trick to provide a symbols mapping while printing (and not putting it in every object)
-    
+
     def __init__(self, codename):
         # the rule name
         #assert type(codename)==int
@@ -311,10 +311,26 @@
         pass
 
     def __str__(self):
-        return self.display(0, GrammarElement.symbols )
+        # XXX: remove me after debug
+        symbols = {}
+        import pytoken
+        import pysymbol
+        symbols.update( pysymbol._cpython_symbols.sym_name )
+        symbols.update( pytoken.tok_name )
+        
+        return self.display(0, symbols )
+#        return self.display(0, GrammarElement.symbols )
 
     def __repr__(self):
-        return self.display(0, GrammarElement.symbols )
+        # XXX: remove me after debug
+        symbols = {}
+        import pytoken
+        import pysymbol
+        symbols.update( pysymbol._cpython_symbols.sym_name )
+        symbols.update( pytoken.tok_name )
+        
+        return self.display(0, symbols )
+#        return self.display(0, GrammarElement.symbols )
 
     def display(self, level=0, symbols={}):
         """Helper function used to represent the grammar.
@@ -364,15 +380,15 @@
 
 
 class GrammarProxy(GrammarElement):
-    def __init__(self, rule_name ):
-        GrammarElement.__init__(self, -1)
+    def __init__(self, rule_name, codename=-1 ):
+        GrammarElement.__init__(self, codename )
         self.rule_name = rule_name
         self.object = None
 
     def display(self, level=0, symbols={}):
         """Helper function used to represent the grammar.
         mostly used for debugging the grammar itself"""
-        name = symbols.get(self.rule_name,str(self.rule_name))
+        name = symbols.get(self.codename, self.rule_name)
         repr = "Proxy("+name
         if self.object:
             repr+=","+self.object.display(level=1,symbols=symbols)

Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/pythonparse.py	Mon Jan 23 18:14:22 2006
@@ -20,9 +20,9 @@
 
 class PythonParser(object):
     """Wrapper class for python grammar"""
-    def __init__(self, grammar_builder):
-        self.items = grammar_builder.items
-        self.rules = grammar_builder.rules
+    def __init__(self, rules, items ):
+        self.items = items
+        self.rules = rules
         # Build first sets for each rule (including anonymous ones)
         grammar.build_first_sets(self.items)
 
@@ -125,11 +125,21 @@
     grammar.DEBUG = 0
     gram = ebnfparse.parse_grammar( file(fname) )
     grammar.DEBUG = level
-    parser = PythonParser( gram )
+    parser = PythonParser( gram.rules, gram.items )
     return parser
 
+def python_grammar_dyn(fname):
+    """Loads the grammar using the 'dynamic' rpython parser"""
+    _grammar_file = file(fname)
+    ebnfbuilder = ebnfparse.parse_grammar_text( file(fname).read() )
+    ebnfbuilder.resolve_rules()
+    parser = PythonParser( ebnfbuilder.root_rules, ebnfbuilder.all_rules )
+    return parser
+
+
 debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
 PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR )
+PYTHON_PARSER_DYN = python_grammar_dyn( PYTHON_GRAMMAR )
 
 def reload_grammar(version):
     """helper function to test with pypy different grammars"""

Modified: pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py	(original)
+++ pypy/branch/ast-experiments/pypy/interpreter/pyparser/test/test_astbuilder.py	Mon Jan 23 18:14:22 2006
@@ -1,6 +1,6 @@
 import os
 
-from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER_DYN as PYTHON_PARSER
 from pypy.interpreter.pyparser.astbuilder import AstBuilder
 from pypy.interpreter.pyparser.pythonutil import ast_from_input
 from pypy.interpreter.stablecompiler.transformer import Transformer



More information about the Pypy-commit mailing list