[pypy-svn] r32634 - in pypy/dist/pypy/interpreter/pyparser: . data
ac at codespeak.net
ac at codespeak.net
Mon Sep 25 16:30:36 CEST 2006
Author: ac
Date: Mon Sep 25 16:30:35 2006
New Revision: 32634
Removed:
pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a_with
Modified:
pypy/dist/pypy/interpreter/pyparser/astbuilder.py
pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a
pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
pypy/dist/pypy/interpreter/pyparser/grammar.py
pypy/dist/pypy/interpreter/pyparser/pythonparse.py
Log:
Replace a hack using two grammars, one including the with statement and one without it, with another hack only using one grammar.
Modified: pypy/dist/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/astbuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Mon Sep 25 16:30:35 2006
@@ -11,7 +11,6 @@
from pypy.interpreter.pyparser.parsestring import parsestr
sym = pythonparse.PYTHON_PARSER.symbols
-sym_with = pythonparse.PYTHON_PARSER.with_grammar.symbols
DEBUG_MODE = 0
@@ -1362,7 +1361,8 @@
if name == 'with_statement':
# found from __future__ import with_statement
if not builder.with_enabled:
- raise pythonparse.AlternateGrammarException()
+ builder.enable_with()
+ #raise pythonparse.AlternateGrammarException()
builder.push(ast.From(from_name, names, atoms[0].lineno))
@@ -1537,15 +1537,12 @@
'exprlist' : build_exprlist,
'decorator' : build_decorator,
'eval_input' : build_eval_input,
+ 'with_stmt' : build_with_stmt,
}
# Build two almost identical ASTRULES dictionaries
ASTRULES = dict([(sym[key], value) for (key, value) in
ASTRULES_Template.iteritems()])
-
-ASTRULES_Template['with_stmt'] = build_with_stmt
-ASTRULES_with = dict([(sym_with[key], value) for (key, value) in
- (ASTRULES_Template).iteritems()])
del ASTRULES_Template
## Stack elements definitions ###################################
@@ -1684,12 +1681,13 @@
self.rule_stack = []
self.space = space
self.source_encoding = None
- self.ASTRULES = ASTRULES
self.with_enabled = False
def enable_with(self):
- self.ASTRULES = ASTRULES_with
+ if self.with_enabled:
+ return
self.with_enabled = True
+ self.keywords.update({'with':None, }) # 'as': None})
def context(self):
return AstBuilderContext(self.rule_stack)
@@ -1729,7 +1727,7 @@
if rule.is_root():
## if DEBUG_MODE:
## print "ALT:", sym.sym_name[rule.codename], self.rule_stack
- builder_func = self.ASTRULES.get(rule.codename, None)
+ builder_func = ASTRULES.get(rule.codename, None)
if builder_func:
builder_func(self, 1)
else:
@@ -1750,7 +1748,7 @@
if rule.is_root():
## if DEBUG_MODE:
## print "SEQ:", sym.sym_name[rule.codename]
- builder_func = self.ASTRULES.get(rule.codename, None)
+ builder_func = ASTRULES.get(rule.codename, None)
if builder_func:
# print "REDUCING SEQUENCE %s" % sym.sym_name[rule.codename]
builder_func(self, elts_number)
Modified: pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a (original)
+++ pypy/dist/pypy/interpreter/pyparser/data/Grammar2.5a Mon Sep 25 16:30:35 2006
@@ -63,12 +63,13 @@
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break
['else' ':' suite] | 'try' ':' suite 'finally' ':' suite)
+with_stmt: 'with' test [ NAME expr ] ':' suite
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test [',' test]]
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
Modified: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/ebnfparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py Mon Sep 25 16:30:35 2006
@@ -44,16 +44,17 @@
else:
# error unknown or negative integer
"""
+
ctx = source.context()
tk = source.next()
if tk.codename==self.codename:
- if tk.value not in self.keywords:
+ if tk.value not in builder.keywords:
ret = builder.token( tk.codename, tk.value, source )
return self.debug_return( ret, tk.codename, tk.value )
source.restore( ctx )
return 0
- def match_token(self, other):
+ def match_token(self, builder, other):
"""special case of match token for tokens which are really keywords
"""
if not isinstance(other, Token):
@@ -62,7 +63,7 @@
return False
if other.codename != self.codename:
return False
- if other.value in self.keywords:
+ if other.value in builder.keywords:
return False
return True
Modified: pypy/dist/pypy/interpreter/pyparser/grammar.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/grammar.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/grammar.py Mon Sep 25 16:30:35 2006
@@ -142,6 +142,7 @@
class BaseGrammarBuilder(AbstractBuilder):
"""Base/default class for a builder"""
+ keywords = None
def __init__(self, rules=None, debug=0, symbols={} ):
AbstractBuilder.__init__(self, rules, debug, symbols )
# stacks contain different objects depending on the builder class
@@ -246,7 +247,7 @@
token = source.peek()
if self._trace:
pos1 = source.get_pos()
- in_first_set = self.match_first_set(token)
+ in_first_set = self.match_first_set(builder, token)
if not in_first_set: # and not EmptyToken in self.first_set:
if EmptyToken in self.first_set:
ret = builder.sequence(self, source, 0 )
@@ -331,12 +332,12 @@
# XXX: first_set could probably be implemented with sets
return []
- def match_first_set(self, other):
+ def match_first_set(self, builder, other):
"""matching is not equality:
token('NAME','x') matches token('NAME',None)
"""
for tk in self.first_set:
- if tk.match_token( other ):
+ if tk.match_token( builder, other ):
return True
return False
@@ -377,7 +378,7 @@
# to see if this solve our problems with infinite recursion
for rule in self.args:
if USE_LOOKAHEAD:
- if not rule.match_first_set(tok) and EmptyToken not in rule.first_set:
+ if not rule.match_first_set(builder, tok) and EmptyToken not in rule.first_set:
if self._trace:
print "Skipping impossible rule: %s" % (rule,)
continue
@@ -642,6 +643,10 @@
else:
# error unknown or negative integer
"""
+ if (self.value is not None and builder.keywords is not None
+ and self.value not in builder.keywords):
+ return 0
+
ctx = source.context()
tk = source.next()
if tk.codename == self.codename:
@@ -664,7 +669,7 @@
return "<%s>=='%s'" % (name, self.value)
- def match_token(self, other):
+ def match_token(self, builder, other):
"""convenience '==' implementation, this is *not* a *real* equality test
a Token instance can be compared to:
- another Token instance in which case all fields (name and value)
@@ -676,7 +681,10 @@
raise RuntimeError("Unexpected token type %r" % other)
if other is EmptyToken:
return False
- res = other.codename == self.codename and self.value in [None, other.value]
+ if (self.value is not None and builder.keywords is not None
+ and self.value not in builder.keywords):
+ return False
+ res = other.codename == self.codename and self.value in [None, other.value]
#print "matching", self, other, res
return res
Modified: pypy/dist/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py Mon Sep 25 16:30:35 2006
@@ -31,7 +31,11 @@
grammar.build_first_sets(self.items)
self.symbols = grammar_builder.symbols
self.with_grammar = None
-
+ self.keywords = dict.fromkeys(grammar_builder.keywords)
+ # Only when with_statement is enabled
+ self.keywords.pop('with', None)
+ self.keywords.pop('as', None)
+
def parse_source(self, textsrc, goal, builder, flags=0):
"""Parse a python source according to goal"""
# Detect source encoding.
@@ -52,22 +56,14 @@
def parse_lines(self, lines, goal, builder, flags=0):
- if (self.with_grammar is not None and # don't recurse into ourself
- flags & CO_FUTURE_WITH_STATEMENT):
+ builder.keywords = self.keywords.copy()
+ if flags & CO_FUTURE_WITH_STATEMENT:
builder.enable_with()
- return self.with_grammar.parse_lines(lines, goal, builder, flags)
goalnumber = self.symbols.sym_values[goal]
target = self.rules[goalnumber]
src = Source(lines, flags)
- result = None
- try:
- result = target.match(src, builder)
- except AlternateGrammarException: # handle from __future__ import with_statement
- if self.with_grammar is not None:
- builder.enable_with()
- return self.with_grammar.parse_lines(lines, goal, builder, flags)
- if not result:
+ if not target.match(src, builder):
line, lineno = src.debug()
# XXX needs better error messages
raise SyntaxError("invalid syntax", lineno, -1, line)
@@ -148,8 +144,8 @@
return parser
debug_print( "Loading grammar %s" % PYTHON_GRAMMAR )
-PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR )
-PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
+PYTHON_PARSER = python_grammar( PYTHON_GRAMMAR)
+#PYTHON_PARSER.with_grammar = python_grammar( PYTHON_GRAMMAR + '_with' )
def reload_grammar(version):
"""helper function to test with pypy different grammars"""
More information about the Pypy-commit
mailing list