[Python-checkins] r52867 - sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py
guido.van.rossum
python-checkins at python.org
Thu Nov 30 04:29:40 CET 2006
Author: guido.van.rossum
Date: Thu Nov 30 04:29:39 2006
New Revision: 52867
Modified:
sandbox/trunk/2to3/play.py
sandbox/trunk/2to3/pynode.py
Log:
Lose lots of unused code from pynode.py.
Modified: sandbox/trunk/2to3/play.py
==============================================================================
--- sandbox/trunk/2to3/play.py (original)
+++ sandbox/trunk/2to3/play.py Thu Nov 30 04:29:39 2006
@@ -33,7 +33,7 @@
tree = dr.parse_file("example.py", debug=True)
sys.stdout.write(str(tree))
- return
+ return # Comment out to run the complete test suite below
# Process every imported module
for name in sys.modules:
Modified: sandbox/trunk/2to3/pynode.py
==============================================================================
--- sandbox/trunk/2to3/pynode.py (original)
+++ sandbox/trunk/2to3/pynode.py Thu Nov 30 04:29:39 2006
@@ -9,11 +9,6 @@
There is a class or function corresponding to each terminal and
nonterminal symbol.
-The grammar is pieced together from the docstrings of the
-corresponding classes and functions: text between dollar signs is
-assumed to be the right-hand side of the grammar rule corresponding to
-that class or function.
-
We use __slots__ to make the parse tree nodes as small as possible.
NOTE: EVERY CLASS MUST HAVE A __slots__ DEFINITION, EVEN IF EMPTY!
@@ -211,11 +206,6 @@
# Nodes and factory functions for Python grammar
-class VanishingNonterminal(Nonterminal):
- __slots__ = []
- def __new__(cls, context, node):
- return node
-
class GenericSeries(Series):
__slots__ = ["nodes"]
def initseries(self, nodes):
@@ -466,182 +456,6 @@
class file_input(GenericSeries):
__slots__ = []
-
-# Example nodes and factory functions for nonterminal symbols
-# XXX: get rid of these
-
-def Program(context, stmts):
- """Program is a nonterminal with only one non-trivial child.
-
- Grammar: $ Expression ENDMARKER $
-
- """
- return stmts
-
-class Expression(Series):
- "Grammar: $ BinaryExpression ['?' BinaryExpression ':' BinaryExpression] $"
-
- __slots__ = ["test", "left", "right"]
-
- def initseries(self, nodes):
- self.test, self.left, self.right = nodes
-
- def __str__(self):
- return "%s ? %s : %s" % (self.test, self.left, self.right)
-
-class BinaryExpression(Series):
- "Grammar: $ Operand (Operator Operand)* $"
-
- __slots__ = ["left", "op", "right"]
-
- def initseries(self, stuff):
- # Stuff is a list with alternating operands and operators
- if len(stuff) == 3:
- self.left, self.op, self.right = stuff
- return
- assert len(stuff) > 1 and len(stuff) % 2 == 1
- # Find the rightmost lowest-priority operator
- lowest_i = 1
- lowest_op = stuff[lowest_i]
- lowest_pri = lowest_op.priority()
- for i in range(3, len(stuff), 2):
- op = stuff[i]
- pri = op.priority()
- if pri <= lowest_pri:
- lowest_i = i
- lowest_op = op
- lowest_pri = pri
- self.left = self.__class__(self.context, *stuff[:lowest_i])
- self.op = lowest_op
- self.right = self.__class__(self.context, *stuff[lowest_i+1:])
-
- def __str__(self):
- return "(%s %s %s)" % (self.left, self.op, self.right)
-
-def Operand(context, arg):
- """Operand is a nonterminal with one child.
-
- Grammar: $ Atom | UnaryExpression $
-
- """
- return arg
-
-class UnaryExpression(Nonterminal):
- "Grammar: $ UnaryOperator Operand $"
-
- __slots__ = ["op", "arg"]
-
- def __init__(self, context, op, arg):
- self.op = op
- self.arg = arg
-
- def __str__(self):
- return "%s%s" % (self.op, self.arg)
-
-def UnaryOperator(context, op):
- "Grammar: $ 'not' | '-' $"
- return op
-
-class XXXOperator(Nonterminal):
- """Operator.
-
- This has a repr slot and a priority() method.
-
- The __new__ method implements a sort-of singleton pattern: there's
- only one instance per repr value. (Yes, this means the context is
- not useful. Therefore we set it to None.)
-
- Grammar: $ '**' | '*' | '/' | '+' | '-' | '&' | '^' | '|' |
- ['not'] 'in' | '==' | '<' | '>' | '!=' | '<=' | '>=' |
- 'and' | 'or' $
-
- """
-
- __slots__ = ["repr"]
-
- _stretch = False
-
- _cache = {}
-
- def __new__(cls, context, *args):
- repr = " ".join(args)
- # For "not in", the argument should be the string "not in"
- obj = cls._cache.get(repr)
- if obj is None:
- obj = Terminal.__new__(cls, None)
- obj.repr = repr
- return obj
-
- def __str__(self):
- return self.repr
-
- def priority(self):
- return self._priorities[self.repr]
-
- _priorities = {
- "or": 0,
- "and": 1,
- "in": 2,
- "not in": 2,
- "<": 2,
- ">": 2,
- "==": 2,
- "<=": 2,
- ">=": 2,
- "!=": 2,
- "|": 3,
- "^": 4,
- "&": 5,
- "+": 6,
- "-": 6,
- "*": 7,
- "/": 7,
- "**": 8,
- }
-
-def Atom(context, arg):
- """ Grammar: $ NAME | STRING | NUMBER | ParenthesizedExpression $
- """
- return arg
-
-def ParenthesizedExpression(context, expr):
- "Grammar: $ '(' Expression ')' $"
- return expr
-
-
-# Conversion from concrete to abstract syntax trees
-
-def vanish(context, value):
- return None
-
-token_mapping = {
- # Tokens that become parse tree nodes
- # (token.NAME is special-cased in the code)
- token.NUMBER: Number,
- token.STRING: String,
-
-## # Tokens that vanish
-## token.DOT: vanish,
-## token.LPAR: vanish,
-## token.RPAR: vanish,
-## token.COLON: vanish,
-## token.COMMA: vanish,
-## token.EQUAL: vanish,
-## token.DEDENT: vanish,
-## token.INDENT: vanish,
-## token.LBRACE: vanish,
-## token.RBRACE: vanish,
-## token.NEWLINE: vanish,
-## token.ENDMARKER: vanish,
-## grammar.QUESTIONMARK: vanish,
-
- # All other tokens return the token's string value (e.g. "+")
- }
-
-vanishing_keywords = {
- # E.g. "def": True,
- }
-
def convert(grammar, node):
type, value, context, children = node
# Is it a non-terminal symbol?
@@ -667,47 +481,12 @@
if type == token.NAME:
# Name or keyword. Special-case the snot out of this.
if value in grammar.keywords:
- # Keywords become strings, like operators
- if value in vanishing_keywords:
- return None
- else:
- return Token(context, value)
+ # Keywords become Tokens
+ return Token(context, value)
else:
return Name(context, value)
- # Look for a handler in the token_mapping table.
assert type in token.tok_name
- factory = token_mapping.get(type)
- if factory:
- return factory(context, value)
- else:
- return Token(context, value)
-
-
-# Support code
-
-def generate_grammar(stream):
- """Extract the grammar rules from this module's doc strings."""
- import re
- from types import ModuleType
- lines = []
- startline = None
- for name, obj in globals().items():
- if hasattr(obj, "__doc__") and not isinstance(obj, ModuleType):
- m = re.search(r"Grammar:\s*\$([^$]+)\$", obj.__doc__ or "")
- if m:
- rule = obj.__name__, " ".join(m.group(1).split())
- if rule[0] == "Program":
- assert not startline
- startline = rule
- else:
- lines.append(rule)
- lines.sort()
- # The start symbol *must* be the first rule
- lines.insert(0, startline)
- for name, rhs in lines:
- stream.write("%s: %s\n" % (name, rhs))
-
-if __name__ == '__main__':
- import sys
- generate_grammar(sys.stdout)
+
+ # Operators become Tokens
+ return Token(context, value)
More information about the Python-checkins
mailing list