[pypy-svn] r14349 - in pypy/dist/pypy/interpreter: pyparser pyparser/test stablecompiler
ludal at codespeak.net
ludal at codespeak.net
Wed Jul 6 18:49:54 CEST 2005
Author: ludal
Date: Wed Jul 6 18:49:51 2005
New Revision: 14349
Added:
pypy/dist/pypy/interpreter/pyparser/astbuilder.py
- copied, changed from r14156, pypy/branch/pycompiler/module/recparser/astbuilder.py
pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
Modified:
pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
pypy/dist/pypy/interpreter/stablecompiler/transformer.py
Log:
start working on an astbuilder
Copied: pypy/dist/pypy/interpreter/pyparser/astbuilder.py (from r14156, pypy/branch/pycompiler/module/recparser/astbuilder.py)
==============================================================================
--- pypy/branch/pycompiler/module/recparser/astbuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/astbuilder.py Wed Jul 6 18:49:51 2005
@@ -1,48 +1,191 @@
from grammar import BaseGrammarBuilder
-from compiler.ast import nodes, TokenNode
-from compiler.astfactory import factory_functions, group_factory, syntaxnode_factory
+import pypy.interpreter.astcompiler.ast as ast
+def get_atoms( builder, nb ):
+ L = []
+ i = nb
+ while i>0:
+ obj = builder.pop()
+ if isinstance(obj,RuleObject):
+ i+=obj.count
+ else:
+ L.append( obj )
+ i -= 1
+ L.reverse()
+ return L
+
+
+def build_single_input( builder, nb ):
+ pass
+
+def build_atom( builder, nb ):
+ L = get_atoms( builder, nb )
+ top = L[0]
+ if isinstance(top, TokenObject):
+ if top.name == "(":
+ builder. ast.Tuple(L[1:-1], top.line)
+ elif top.name == "[":
+ builder.push( ast.List( L[1:-1], top.line) )
+ elif top.name == "{":
+ builder.push( ast.Dict( L[1:-1], top.line) )
+ elif top.name == "NAME":
+ builder.push( ast.Name(top.value) )
+ elif top.name == "NUMBER":
+ builder.push( ast.Const(eval(top.value)) )
+ else:
+ raise ValueError, "unexpected tokens (%d): %s" % (nb,[ str(i) for i in L] )
+
+
+def build_power( builder, nb ):
+ L = get_atoms( builder, nb )
+ if len(L) == 1:
+ builder.push( L[0] )
+ elif len(L) == 3:
+ builder.push( ast.Power( [ L[0], L[2] ] ) )
+ else:
+ raise ValueError, "unexpected tokens: %s" % L
+
+def build_factor( builder, nb ):
+ L = get_atoms( builder, nb )
+ if len(L) == 1:
+ builder.push( L[0] )
+ elif len(L) == 2 and isinstance(L[0],TokenObject):
+ if L[0].name == "+":
+ builder.push( ast.UnaryAdd( L[1] ) )
+ if L[0].name == "-":
+ builder.push( ast.UnarySub( L[1] ) )
+ if L[0].name == "~":
+ builder.push( ast.Invert( L[1] ) )
+
+def build_term( builder, nb ):
+ L = get_atoms( builder, nb )
+ l = len(L)
+ left = L[0]
+ for i in range(2,l,2):
+ right = L[i]
+ op = L[i-1].name
+ if op == "*":
+ left = ast.Mul( [ left, right ] )
+ elif op == "/":
+ left = ast.Div( [ left, right ] )
+ elif op == "%":
+ left = ast.Mod( [ left, right ] )
+ elif op == "//":
+ left = ast.FloorDiv( [ left, right ] )
+ else:
+ raise ValueError, "unexpected token: %s" % L[i-1]
+ builder.push( left )
+
+def build_arith_expr( builder, nb ):
+ L = get_atoms( builder, nb )
+ l = len(L)
+ left = L[0]
+ for i in range(2,l,2):
+ right = L[i]
+ op = L[i-1].name
+ if op == "+":
+ left = ast.Add( [ left, right ] )
+ elif op == "-":
+ left = ast.Sub( [ left, right ] )
+ else:
+ raise ValueError, "unexpected token: %s : %s" % L[i-1]
+ builder.push( left )
+
+def build_shift_expr( builder, nb ):
+ L = get_atoms( builder, nb )
+ l = len(L)
+ left = L[0]
+ for i in range(2,l,2):
+ right = L[i]
+ op = L[i-1].name
+ if op == "<<":
+ left = ast.LeftShift( [ left, right ] )
+ elif op == ">>":
+ left = ast.RightShift( [ left, right ] )
+ else:
+ raise ValueError, "unexpected token: %s : %s" % L[i-1]
+ builder.push( left )
+
+
+ASTRULES = {
+# "single_input" : build_single_input,
+ "atom" : build_atom,
+ "power" : build_power,
+ "factor" : build_factor,
+ "term" : build_term,
+ "arith_expr" : build_arith_expr,
+ "shift_expr" : build_shift_expr,
+ }
+
+class RuleObject(ast.Node):
+ """A simple object used to wrap a rule or token"""
+ def __init__(self, name, count, src ):
+ self.name = name
+ self.count = count
+ self.line = 0 # src.getline()
+ self.col = 0 # src.getcol()
+
+ def __str__(self):
+ return "<Rule: %s>" % (self.name,)
+
+class TokenObject(ast.Node):
+ """A simple object used to wrap a rule or token"""
+ def __init__(self, name, value, src ):
+ self.name = name
+ self.value = value
+ self.count = 0
+ self.line = 0 # src.getline()
+ self.col = 0 # src.getcol()
+
+ def __str__(self):
+ return "<Token: %s=%s>" % (self.name, self.value)
+
class AstBuilder(BaseGrammarBuilder):
"""A builder that directly produce the AST"""
def __init__( self, rules=None, debug=0 ):
BaseGrammarBuilder.__init__(self, rules, debug )
+ self.rule_stack = []
+
+ def pop(self):
+ return self.rule_stack.pop(-1)
+
+ def push(self, obj):
+ if not isinstance(obj, RuleObject) and not isinstance(obj, TokenObject):
+ print "Pushed:", str(obj), len(self.rule_stack)
+ self.rule_stack.append( obj )
+
+ def push_tok(self, name, value, src ):
+ self.push( TokenObject( name, value, src ) )
- def top(self, n=1):
- toplist = []
- for node in self.stack[-n:]:
- toplist += node.expand()
- return toplist
+ def push_rule(self, name, count, src ):
+ self.push( RuleObject( name, count, src ) )
def alternative( self, rule, source ):
# Do nothing, keep rule on top of the stack
if rule.is_root():
- ast_factory = factory_functions.get( rule.name, syntaxnode_factory )
- elems = self.top()
- node = ast_factory( rule.name, source, elems )
- self.stack[-1] = node
- if self.debug:
- self.stack[-1].dumpstr()
+ print "ALT:",rule.name
+ F = ASTRULES.get(rule.name)
+ if F:
+ F( self, 1 )
+ else:
+ self.push_rule( rule.name, 1, source )
return True
def sequence(self, rule, source, elts_number):
""" """
- items = self.top( elts_number )
if rule.is_root():
- ast_factory = factory_functions.get( rule.name, syntaxnode_factory )
- else:
- ast_factory = group_factory
-
- node = ast_factory( rule.name, source, items )
- # replace N elements with 1 element regrouping them
- if elts_number >= 1:
- self.stack[-elts_number:] = node
+ print "SEQ:", rule.name
+ F = ASTRULES.get(rule.name)
+ if F:
+ F( self, elts_number )
else:
- self.stack.append(node)
+ self.push_rule( rule.name, elts_number, source )
return True
def token(self, name, value, source):
- self.stack.append(TokenNode(name, source, value))
+ print "TOK:", name, value
+ self.push_tok( name, value, source )
return True
Modified: pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/syntaxtree.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/syntaxtree.py Wed Jul 6 18:49:51 2005
@@ -63,12 +63,6 @@
}
NT_OFFSET = token.NT_OFFSET
-SYMBOLS = {}
-# copies the numerical mapping between symbol name and symbol value
-# into SYMBOLS
-for k, v in symbol.sym_name.items():
- SYMBOLS[v] = k
-SYMBOLS['UNKNOWN'] = -1
class SyntaxNode(object):
Added: pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_astbuilder.py Wed Jul 6 18:49:51 2005
@@ -0,0 +1,26 @@
+
+from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser.astbuilder import AstBuilder
+from pypy.interpreter.pyparser.pythonutil import ast_from_input
+from pypy.interpreter.stablecompiler.transformer import Transformer
+import py.test
+
+expr1 = "x = a + 1"
+
+
+
+def ast_parse_expr( expr ):
+ builder = AstBuilder()
+ PYTHON_PARSER.parse_source( expr, "single_input", builder )
+ return builder
+
+def tuple_parse_expr( expr ):
+ t = Transformer()
+ return ast_from_input( expr, "single", t )
+
+def test_expr1():
+ py.test.skip("work in progress")
+ r1 = ast_parse_expr( expr1 )
+ ast = tuple_parse_expr( expr1 )
+ print ast
+
Modified: pypy/dist/pypy/interpreter/stablecompiler/transformer.py
==============================================================================
--- pypy/dist/pypy/interpreter/stablecompiler/transformer.py (original)
+++ pypy/dist/pypy/interpreter/stablecompiler/transformer.py Wed Jul 6 18:49:51 2005
@@ -25,13 +25,13 @@
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+# make sure we import the parser with the correct grammar
+import pypy.interpreter.pyparser.pythonparse
from pypy.interpreter.stablecompiler.ast import *
import parser
import symbol
import token
import sys
-# make sure we import the parser with the correct grammar
-import pypy.interpreter.pyparser.pythonparse
class WalkerError(StandardError):
pass
More information about the Pypy-commit
mailing list