[pypy-svn] r32634 - in pypy/dist/pypy/interpreter/pyparser: . data

ac at codespeak.net ac at codespeak.net
Mon Sep 25 16:30:36 CEST 2006


Author: ac
Date: Mon Sep 25 16:30:35 2006
New Revision: 32634

Removed:
   pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a_with
Modified:
   pypy/dist/pypy/interpreter/pyparser/astbuilder.py
   pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a
   pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
   pypy/dist/pypy/interpreter/pyparser/grammar.py
   pypy/dist/pypy/interpreter/pyparser/pythonparse.py
Log:
Replace a hack using two grammars, one including the with statement and one without it, with another hack only using one grammar.



Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/astbuilder.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py	Mon Sep 25 16:30:35 2006
@@ -11,7 +11,6 @@
 from pypy.interpreter.pyparser.parsestring import parsestr
 
 sym      = pythonparse.PYTHON_PARSER.symbols
-sym_with = pythonparse.PYTHON_PARSER.with_grammar.symbols
 
 DEBUG_MODE = 0
 
@@ -1362,7 +1361,8 @@
             if name == 'with_statement':
                 # found from __future__ import with_statement
                 if not builder.with_enabled:
-                    raise pythonparse.AlternateGrammarException()
+                    builder.enable_with()
+                    #raise pythonparse.AlternateGrammarException()
     builder.push(ast.From(from_name, names, atoms[0].lineno))
 
 
@@ -1537,15 +1537,12 @@
     'exprlist' : build_exprlist,
     'decorator' : build_decorator,
     'eval_input' : build_eval_input,
+    'with_stmt' : build_with_stmt,
     }
 
 # Build two almost identical ASTRULES dictionaries
 ASTRULES      = dict([(sym[key], value) for (key, value) in
                       ASTRULES_Template.iteritems()])
-
-ASTRULES_Template['with_stmt'] = build_with_stmt
-ASTRULES_with = dict([(sym_with[key], value) for (key, value) in
-                      (ASTRULES_Template).iteritems()])
 del ASTRULES_Template
 
 ## Stack elements definitions ###################################
@@ -1684,12 +1681,13 @@
         self.rule_stack = []
         self.space = space
         self.source_encoding = None
-        self.ASTRULES = ASTRULES
         self.with_enabled = False
 
     def enable_with(self):
-        self.ASTRULES = ASTRULES_with
+        if self.with_enabled:
+            return
         self.with_enabled = True
+        self.keywords.update({'with':None, }) #  'as': None})
 
     def context(self):
         return AstBuilderContext(self.rule_stack)
@@ -1729,7 +1727,7 @@
         if rule.is_root():
 ##             if DEBUG_MODE:
 ##                 print "ALT:", sym.sym_name[rule.codename], self.rule_stack
-            builder_func = self.ASTRULES.get(rule.codename, None)
+            builder_func = ASTRULES.get(rule.codename, None)
             if builder_func:
                 builder_func(self, 1)
             else:
@@ -1750,7 +1748,7 @@
         if rule.is_root():
 ##             if DEBUG_MODE:
 ##                 print "SEQ:", sym.sym_name[rule.codename]
-            builder_func = self.ASTRULES.get(rule.codename, None)
+            builder_func = ASTRULES.get(rule.codename, None)
             if builder_func:
                 # print "REDUCING SEQUENCE %s" % sym.sym_name[rule.codename]
                 builder_func(self, elts_number)

Modified: pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a	(original)
+++ pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a	Mon Sep 25 16:30:35 2006
@@ -63,12 +63,13 @@
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break
            ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite)
+with_stmt: 'with' test [ NAME expr ] ':' suite
 # NB compile.c makes sure that the default except clause is last
 except_clause: 'except' [test [',' test]]
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT

Modified: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/ebnfparse.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py	Mon Sep 25 16:30:35 2006
@@ -44,16 +44,17 @@
         else:
             # error unknown or negative integer
         """
+        
         ctx = source.context()
         tk = source.next()
         if tk.codename==self.codename:
-            if tk.value not in self.keywords:
+            if tk.value not in builder.keywords:
                 ret = builder.token( tk.codename, tk.value, source )
                 return self.debug_return( ret, tk.codename, tk.value )
         source.restore( ctx )
         return 0
         
-    def match_token(self, other):
+    def match_token(self, builder, other):
         """special case of match token for tokens which are really keywords
         """
         if not isinstance(other, Token):
@@ -62,7 +63,7 @@
             return False
         if other.codename != self.codename:
             return False
-        if other.value in self.keywords:
+        if other.value in builder.keywords:
             return False
         return True
 

Modified: pypy/dist/pypy/interpreter/pyparser/grammar.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/grammar.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/grammar.py	Mon Sep 25 16:30:35 2006
@@ -142,6 +142,7 @@
 
 class BaseGrammarBuilder(AbstractBuilder):
     """Base/default class for a builder"""
+    keywords = None
     def __init__(self, rules=None, debug=0, symbols={} ):
         AbstractBuilder.__init__(self, rules, debug, symbols )
         # stacks contain different objects depending on the builder class
@@ -246,7 +247,7 @@
         token = source.peek()
         if self._trace:
             pos1 = source.get_pos()
-        in_first_set = self.match_first_set(token)
+        in_first_set = self.match_first_set(builder, token)
         if not in_first_set: # and not EmptyToken in self.first_set:
             if EmptyToken in self.first_set:
                 ret = builder.sequence(self, source, 0 )
@@ -331,12 +332,12 @@
         # XXX: first_set could probably be implemented with sets
         return []
 
-    def match_first_set(self, other):
+    def match_first_set(self, builder, other):
         """matching is not equality:
         token('NAME','x') matches token('NAME',None)
         """
         for tk in self.first_set:
-            if tk.match_token( other ):
+            if tk.match_token( builder, other ):
                 return True
         return False
 
@@ -377,7 +378,7 @@
         # to see if this solve our problems with infinite recursion
         for rule in self.args:
             if USE_LOOKAHEAD:
-                if not rule.match_first_set(tok) and EmptyToken not in rule.first_set:
+                if not rule.match_first_set(builder, tok) and EmptyToken not in rule.first_set:
                     if self._trace:
                         print "Skipping impossible rule: %s" % (rule,)
                     continue
@@ -642,6 +643,10 @@
         else:
             # error unknown or negative integer
         """
+        if (self.value is not None and builder.keywords is not None
+            and self.value not in builder.keywords):
+            return 0
+        
         ctx = source.context()
         tk = source.next()
         if tk.codename == self.codename:
@@ -664,7 +669,7 @@
             return "<%s>=='%s'" % (name, self.value)
     
 
-    def match_token(self, other):
+    def match_token(self, builder, other):
         """convenience '==' implementation, this is *not* a *real* equality test
         a Token instance can be compared to:
          - another Token instance in which case all fields (name and value)
@@ -676,7 +681,10 @@
             raise RuntimeError("Unexpected token type %r" % other)
         if other is EmptyToken:
             return False
-        res = other.codename == self.codename and self.value in [None, other.value]
+        if (self.value is not None and builder.keywords is not None
+            and self.value not in builder.keywords):
+            return False
+        res = other.codename == self.codename and self.value in [None, other.value] 
         #print "matching", self, other, res
         return res
     

Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py	Mon Sep 25 16:30:35 2006
@@ -31,7 +31,11 @@
         grammar.build_first_sets(self.items)
         self.symbols = grammar_builder.symbols
         self.with_grammar = None
-
+        self.keywords = dict.fromkeys(grammar_builder.keywords)
+        # Only when with_statement is enabled
+        self.keywords.pop('with', None)
+        self.keywords.pop('as', None)
+        
     def parse_source(self, textsrc, goal, builder, flags=0):
         """Parse a python source according to goal"""
         # Detect source encoding.
@@ -52,22 +56,14 @@
 
 
     def parse_lines(self, lines, goal, builder, flags=0):
-        if (self.with_grammar is not None and # don't recurse into ourself
-            flags & CO_FUTURE_WITH_STATEMENT):
+        builder.keywords = self.keywords.copy()
+        if flags & CO_FUTURE_WITH_STATEMENT:
             builder.enable_with()
-            return self.with_grammar.parse_lines(lines, goal, builder, flags)
         goalnumber = self.symbols.sym_values[goal]
         target = self.rules[goalnumber]
         src = Source(lines, flags)
         
-        result = None
-        try:
-            result = target.match(src, builder)
-        except AlternateGrammarException: # handle from __future__ import with_statement
-            if self.with_grammar is not None:
-                builder.enable_with()
-                return self.with_grammar.parse_lines(lines, goal, builder, flags)
-        if not result:
+        if not target.match(src, builder):
             line, lineno = src.debug()
             # XXX needs better error messages
             raise SyntaxError("invalid syntax", lineno, -1, line)
@@ -148,8 +144,8 @@
     return parser
 
 debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
-PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR )
-PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
+PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR)
+#PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
 
 def reload_grammar(version):
     """helper function to test with pypy different grammars"""



More information about the Pypy-commit mailing list