Mailman 3 November 2006 - Python-checkins

r52871 - sandbox/trunk/2to3/play.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 20:19:10 2006 New Revision: 52871 Modified: sandbox/trunk/2to3/play.py Log: Show diffs for example.py too. Modified: sandbox/trunk/2to3/play.py ============================================================================== --- sandbox/trunk/2to3/play.py (original) +++ sandbox/trunk/2to3/play.py Thu Nov 30 20:19:10 2006 @@ -21,9 +21,12 @@ gr = driver.load_grammar("Grammar.txt") dr = driver.Driver(gr, convert=pynode.convert) - tree = dr.parse_file("example.py", debug=True) + fn = "example.py" + tree = dr.parse_file(fn, debug=True) tree.set_parents() sys.stdout.write(str(tree)) + if not diff(fn, tree): + print "No diffs." return # Comment out to run the complete test suite below problems = []

1 0

r52870 - in sandbox/trunk/2to3: example.py pgen2/driver.py tokenize.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 20:11:10 2006 New Revision: 52870 Modified: sandbox/trunk/2to3/example.py sandbox/trunk/2to3/pgen2/driver.py sandbox/trunk/2to3/tokenize.py Log: Tweak the prefix computation to handle alternative horizontal whitespace (\t, \f) properly. Change tokenize.py to return a single (NL, "\\\n") token for a backslash continuation line. Modified: sandbox/trunk/2to3/example.py ============================================================================== --- sandbox/trunk/2to3/example.py (original) +++ sandbox/trunk/2to3/example.py Thu Nov 30 20:11:10 2006 @@ -1,8 +1,13 @@ #!/usr/bin/python + # comment indented by tab """Docstring.""" d = {"x": 42} if d.has_key("x") or d.has_key("y"): - print d["x"] + print d["x"] + \ + 42 + +def foo(): + pass # body indented by tab # This is the last line. Modified: sandbox/trunk/2to3/pgen2/driver.py ============================================================================== --- sandbox/trunk/2to3/pgen2/driver.py (original) +++ sandbox/trunk/2to3/pgen2/driver.py Thu Nov 30 20:11:10 2006 @@ -32,7 +32,7 @@ logger = logging.getLogger() self.logger = logger self.convert = convert - + def parse_stream_raw(self, stream, debug=False): """Parse a stream and return the concrete syntax tree.""" p = parse.Parser(self.grammar, self.convert) @@ -51,7 +51,7 @@ lineno = s_lineno column = 0 if column < s_column: - prefix += " " * (s_column - column) + prefix += line_text[column:s_column] column = s_column if type in (tokenize.COMMENT, tokenize.NL): prefix += value @@ -63,7 +63,8 @@ if type == token.OP: type = grammar.opmap[value] if debug: - self.logger.debug("%s %r", token.tok_name[type], value) + self.logger.debug("%s %r (prefix=%r)", + token.tok_name[type], value, prefix) if p.addtoken(type, value, (prefix, start)): if debug: self.logger.debug("Stop.") Modified: sandbox/trunk/2to3/tokenize.py ============================================================================== --- sandbox/trunk/2to3/tokenize.py (original) +++ sandbox/trunk/2to3/tokenize.py Thu Nov 30 20:11:10 2006 @@ -371,7 +371,7 @@ yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: - yield (NL, initial, spos, (spos[0], spos[1]+1), line) + yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1

1 0

r52869 - sandbox/trunk/2to3/pgen2/astnode.py sandbox/trunk/2to3/pgen2/test.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 19:44:27 2006 New Revision: 52869 Removed: sandbox/trunk/2to3/pgen2/astnode.py sandbox/trunk/2to3/pgen2/test.py Log: Delete stuff we don't need. Deleted: /sandbox/trunk/2to3/pgen2/astnode.py ============================================================================== --- /sandbox/trunk/2to3/pgen2/astnode.py Thu Nov 30 19:44:27 2006 +++ (empty file) @@ -1,434 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Syntax tree node definitions. - -There is a class or function corresponding to each terminal and -nonterminal symbol. - -The grammar is pieced together from the docstrings of the -corresponding classes and functions: text between dollar signs is -assumed to be the right-hand side of the grammar rule corresponding to -that class or function. - -We use __slots__ to make the parse tree nodes as small as possible. - -NOTE: EVERY CLASS MUST HAVE A __slots__ DEFINITION, EVEN IF EMPTY! -(If not, the instances will get a __dict__, defeating the purpose of -__slots__ in our case.) - -""" - -# Python imports -import token -import logging - -from pgen2 import grammar - -logger = logging.getLogger("pgen2.astnode") - -class Node(object): - """Abstract base class for all nodes. - - This has no attributes except a context slot which holds the lone - number (or more detailed context info). In the future this might - change this to several slots (e.g. filename, lineno, column, or - even filename, start_lineno, start_column, end_lineno, - end_column). The context is only referenced by two places: the - part of the code that sticks it it, and the part of the code that - reports errors. - - In order to reduce the amount of boilerplate code, the context is - argument is handled by __new__ rather than __init__. There are - also a few subclasses that override __new__ to sometimes avoid - constructing an instance. - - """ - - __slots__ = ["context"] - - def __new__(cls, context, *rest): - assert cls not in (Node, Nonterminal, Terminal, Constant) - obj = object.__new__(cls) - obj.context = context - return obj - - _stretch = False # Set to true to stretch the repr() vertically - - def __repr__(self, repr=repr): - stretch = self._stretch - r = [self.__class__.__name__] - if stretch: - r.append("(\n ") - else: - r.append("(") # ")" -- crutch for Emacs python-mode :-( - cls = self.__class__ - # Get nearest non-empty slots __slots__. This assumes - # *something* has non-empty __slots__ before we reach object - # (which has no __slots__). The class hierarchy guarantees - # this. - slots = cls.__slots__ - while not slots: - cls = cls.__base__ - slots = cls.__slots__ - first = True - for name in slots: - if name == "context": - continue # Skip this - if first: - first = False - else: - if stretch: - r.append(",\n ") - else: - r.append(", ") - try: - value = getattr(self, name) - except AttributeError: - continue - if stretch and isinstance(value, list): - rr = map(repr, value) - rv = "[" + ",\n ".join(rr) + "]" - else: - rv = repr(value) - if stretch: - rv = rv.replace("\n", "\n ") - r.append(rv) - r.append(")") - return "".join(r) - - def __str__(self): - return self.__repr__(repr=str) - -class Nonterminal(Node): - """Abstract base class for nonterminal symbols. - - Nothing beyond Node. - - """ - - __slots__ = [] - - _stretch = True - -class Terminal(Node): - """Abstract base class for terminal symbols. - - Nothing beyond Node. - - """ - - __slots__ = [] - -class Series(Nonterminal): - """Abstract base class for nonterminals like stmts: stmt+.""" - - __slots__ = [] - - def __new__(cls, context, *nodes): - assert cls is not Series - if len(nodes) == 0: - return None - elif len(nodes) == 1: - return nodes[0] - else: - obj = Nonterminal.__new__(cls, context) - obj.initseries(nodes) - return obj - -class Constant(Terminal): - """Abstract base class for constants (e.g. number or string). - - Attributes: - - repr -- a string giving the token, exactly as read from source - - """ - - __slots__ = ["repr"] - - def __init__(self, context, repr): - self.repr = repr - - def __str__(self): - return self.repr - -# Node classes for terminal symbols - -class Name(Terminal): - """Name (e.g. a variable name or an attribute name). - - Attributes: - - name -- a string giving the name. - - """ - - __slots__ = ["name"] - - def __init__(self, context, name): - self.name = name - - def __str__(self): - return self.name - -class Number(Constant): - """Numeric constant. - - Nothing beyond Constant. - - """ - - __slots__ = [] - -class String(Constant): - """String constant. - - Nothing beyond Constant. - - """ - - __slots__ = [] - -# Example nodes and factory functions for nonterminal symbols - -def Program(context, stmts): - """Program is a nonterminal with only one non-trivial child. - - Grammar: $ Expression ENDMARKER $ - - """ - return stmts - -class Expression(Series): - "Grammar: $ BinaryExpression ['?' BinaryExpression ':' BinaryExpression] $" - - __slots__ = ["test", "left", "right"] - - def initseries(self, nodes): - self.test, self.left, self.right = nodes - - def __str__(self): - return "%s ? %s : %s" % (self.test, self.left, self.right) - -class BinaryExpression(Series): - "Grammar: $ Operand (Operator Operand)* $" - - __slots__ = ["left", "op", "right"] - - def initseries(self, stuff): - # Stuff is a list with alternating operands and operators - if len(stuff) == 3: - self.left, self.op, self.right = stuff - return - assert len(stuff) > 1 and len(stuff) % 2 == 1 - # Find the rightmost lowest-priority operator - lowest_i = 1 - lowest_op = stuff[lowest_i] - lowest_pri = lowest_op.priority() - for i in range(3, len(stuff), 2): - op = stuff[i] - pri = op.priority() - if pri <= lowest_pri: - lowest_i = i - lowest_op = op - lowest_pri = pri - self.left = self.__class__(self.context, *stuff[:lowest_i]) - self.op = lowest_op - self.right = self.__class__(self.context, *stuff[lowest_i+1:]) - - def __str__(self): - return "(%s %s %s)" % (self.left, self.op, self.right) - -def Operand(context, arg): - """Operand is a nonterminal with one child. - - Grammar: $ Atom | UnaryExpression $ - - """ - return arg - -class UnaryExpression(Nonterminal): - "Grammar: $ UnaryOperator Operand $" - - __slots__ = ["op", "arg"] - - def __init__(self, context, op, arg): - self.op = op - self.arg = arg - - def __str__(self): - return "%s%s" % (self.op, self.arg) - -def UnaryOperator(context, op): - "Grammar: $ 'not' | '-' $" - return op - -class Operator(Nonterminal): - """Operator. - - This has a repr slot and a priority() method. - - The __new__ method implements a sort-of singleton pattern: there's - only one instance per repr value. (Yes, this means the context is - not useful. Therefore we set it to None.) - - Grammar: $ '**' | '*' | '/' | '+' | '-' | '&' | '^' | '|' | - ['not'] 'in' | '==' | '<' | '>' | '!=' | '<=' | '>=' | - 'and' | 'or' $ - - """ - - __slots__ = ["repr"] - - _stretch = False - - _cache = {} - - def __new__(cls, context, *args): - repr = " ".join(args) - # For "not in", the argument should be the string "not in" - obj = cls._cache.get(repr) - if obj is None: - obj = Terminal.__new__(cls, None) - obj.repr = repr - return obj - - def __str__(self): - return self.repr - - def priority(self): - return self._priorities[self.repr] - - _priorities = { - "or": 0, - "and": 1, - "in": 2, - "not in": 2, - "<": 2, - ">": 2, - "==": 2, - "<=": 2, - ">=": 2, - "!=": 2, - "|": 3, - "^": 4, - "&": 5, - "+": 6, - "-": 6, - "*": 7, - "/": 7, - "**": 8, - } - -def Atom(context, arg): - """ Grammar: $ NAME | STRING | NUMBER | ParenthesizedExpression $ - """ - return arg - -def ParenthesizedExpression(context, expr): - "Grammar: $ '(' Expression ')' $" - return expr - - -# Conversion from concrete to abstract syntax trees - -def vanish(context, value): - return None - -token_mapping = { - # Tokens that become parse tree nodes - # (token.NAME is special-cased in the code) - token.NUMBER: Number, - token.STRING: String, - - # Tokens that vanish - token.DOT: vanish, - token.LPAR: vanish, - token.RPAR: vanish, - token.COLON: vanish, - token.COMMA: vanish, - token.EQUAL: vanish, - token.DEDENT: vanish, - token.INDENT: vanish, - token.LBRACE: vanish, - token.RBRACE: vanish, - token.NEWLINE: vanish, - token.ENDMARKER: vanish, - - # All other tokens return the token's string value (e.g. "+") - } - -vanishing_keywords = { - # E.g. "def": True, - } - -def convert(grammar, node): - type, value, context, children = node - # Is it a non-terminal symbol? - if type in grammar.number2symbol: - symbol = grammar.number2symbol[type] - factory = globals().get(symbol) - if factory is None: - raise RuntimeError("can't find factory for %s (line %s)" % - (symbol, context)) - # Debug variation: - try: - return factory(context, *children) - except: - logger.debug("%s %s", factory.__name__, "(") - for child in children: - logger.debug("%s %s", "==>", child) - logger.debug(")") - logger.debug("# Did you remember to declare a 'context' arg?") - raise - return factory(context, *children) - - # Must be a terminal symbol. - if type == token.NAME: - # Name or keyword. Special-case the snot out of this. - if value in grammar.keywords: - # Keywords become strings, like operators - if value in vanishing_keywords: - return None - else: - return value - else: - return Name(context, value) - - # Look for a handler in the token_mapping table. - assert type in token.tok_name - factory = token_mapping.get(type) - if factory: - return factory(context, value) - else: - return value - - -# Support code - -def generate_grammar(stream): - """Extract the grammar rules from this module's doc strings.""" - import re - from types import ModuleType - lines = [] - startline = None - for name, obj in globals().items(): - if hasattr(obj, "__doc__") and not isinstance(obj, ModuleType): - m = re.search(r"Grammar:\s*\$([^$]+)\$", obj.__doc__ or "") - if m: - rule = obj.__name__, " ".join(m.group(1).split()) - if rule[0] == "Program": - assert not startline - startline = rule - else: - lines.append(rule) - lines.sort() - # The start symbol *must* be the first rule - lines.insert(0, startline) - for name, rhs in lines: - stream.write("%s: %s\n" % (name, rhs)) - -if __name__ == '__main__': - import sys - generate_grammar(sys.stdout) Deleted: /sandbox/trunk/2to3/pgen2/test.py ============================================================================== --- /sandbox/trunk/2to3/pgen2/test.py Thu Nov 30 19:44:27 2006 +++ (empty file) @@ -1,22 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2006 Python Software Foundation. All Rights Reserved. - -def test(): - import sys - sys.path[0] = ".." - from pgen2 import astnode, driver - f = open("Grammar.txt", "w") - try: - astnode.generate_grammar(f) - finally: - f.close() - sample = "year<=1989 ? ('Modula-3' + ABC) ** 2 : Python" - dr = driver.Driver(driver.load_grammar()) - tree = dr.parse_string(sample, True) - print tree - -if __name__ == "__main__": - test()

1 0

r52868 - in sandbox/trunk/2to3: README example.py fix_has_key.py pgen2/driver.py pgen2/parse.py pgen2/test.py play.py pynode.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 07:15:54 2006 New Revision: 52868 Added: sandbox/trunk/2to3/README (contents, props changed) sandbox/trunk/2to3/fix_has_key.py (contents, props changed) Modified: sandbox/trunk/2to3/example.py sandbox/trunk/2to3/pgen2/driver.py sandbox/trunk/2to3/pgen2/parse.py sandbox/trunk/2to3/pgen2/test.py sandbox/trunk/2to3/play.py (contents, props changed) sandbox/trunk/2to3/pynode.py Log: Got a working 'has_key' refactoring, on a part with ../refactor/has_key.py. Added: sandbox/trunk/2to3/README ============================================================================== --- (empty file) +++ sandbox/trunk/2to3/README Thu Nov 30 07:15:54 2006 @@ -0,0 +1,21 @@ +A refactoring tool for converting Python 2.x code to 3.0. + +This is a prototype! + +Files: + +play.py - test program +pynode.py - parse tree nodes for Python grammar +tokenize.py - modified version of stdlib tokenize.py (1) +fix_has_key.py - refactoring tool changing 'x.has_key(y)' into 'y in x' +example.py - example input +Grammar.txt - Grammar input (same as Grammar/Grammar in Python 2.5) +Grammar.pickle - Pickled grammar tables +pgen2/ - Parser generator and driver (2) + +Notes: + +(1) tokenize.py was modified to yield a NL pseudo-token for backslash + continuations, so the original source can be reproduced exactly. +(2) pgen2 was developed while I was at Elemental Security. It was + modified to suit the needs of this refactoring tool. Modified: sandbox/trunk/2to3/example.py ============================================================================== --- sandbox/trunk/2to3/example.py (original) +++ sandbox/trunk/2to3/example.py Thu Nov 30 07:15:54 2006 @@ -1,8 +1,8 @@ #!/usr/bin/python -"""Docstring. -""" -print (12 + # Hello - # world +"""Docstring.""" + +d = {"x": 42} +if d.has_key("x") or d.has_key("y"): + print d["x"] - 12) # This is the last line. Added: sandbox/trunk/2to3/fix_has_key.py ============================================================================== --- (empty file) +++ sandbox/trunk/2to3/fix_has_key.py Thu Nov 30 07:15:54 2006 @@ -0,0 +1,109 @@ +#!/usr/bin/env python2.5 +# Copyright 2006 Python Software Foundation. All Rights Reserved. + +"""Refactoring tool: change 'x.has_key(y)' into 'y in x'.""" + +__author__ = "Guido van Rossum <guido(a)python.org>" + +# Python imports +import os +import sys +import logging + +import pgen2 +from pgen2 import driver + +import pynode + +logging.basicConfig(level=logging.WARN) + +def main(): + args = sys.argv[1:] or ["example.py"] + + gr = driver.load_grammar("Grammar.txt") + dr = driver.Driver(gr, convert=pynode.convert) + + for fn in args: + print "Parsing", fn + tree = dr.parse_file(fn) + tree.set_parents() + refactor(tree) + diff(fn, tree) + +def refactor(tree): + visit(tree, fix_has_key) + +def visit(node, func): + func(node) + for child in node.get_children(): + visit(child, func) + +# Sample nodes +_context = ("", (0, 0)) +n_dot = pynode.Token(_context, ".") +n_has_key = pynode.Name(_context, "has_key") +n_trailer_has_key = pynode.trailer(_context, n_dot, n_has_key) +n_lpar = pynode.Token(_context, "(") +n_star = pynode.Token(_context, "*") +n_comma = pynode.Token(_context, ",") +n_in = pynode.Token((" ", (0, 0)), "in") + +def fix_has_key(node): + if node != n_trailer_has_key: + return + # XXX Could use more DOM manipulation primitives and matching operations + parent = node.parent + nodes = parent.get_children() + for i, n in enumerate(nodes): + if n is node: + break + else: + print "Can't find node in parent?!" + return + if i+1 >= len(nodes): + return # Nothing follows ".has_key" + if len(nodes) != i+2: + return # Too much follows ".has_key", e.g. ".has_key(x).blah" + next = nodes[i+1] + if not isinstance(next, pynode.trailer): + return # ".has_key" not followed by another trailer + next_children = next.get_children() + if next_children[0] != n_lpar: + return # ".has_key" not followed by "(...)" + if len(next_children) != 3: + return # ".has_key" followed by "()" + argsnode = next_children[1] + arg = argsnode + if isinstance(argsnode, pynode.arglist): + args = argsnode.get_children() + if len(args) > 2: + return # Too many arguments + if len(args) == 2: + if args[0] == n_star: + return # .has_key(*foo) -- you've gotta be kidding! + if args[1] != n_comma: + return # Only .has_key(foo,) expected + arg = args[0] + # Change "X.has_key(Y)" into "Y in X" + arg.set_prefix(nodes[0].get_prefix()) + nodes[0].set_prefix(" ") + new = pynode.comparison(_context, + arg, + n_in, + pynode.power(_context, *nodes[:i])) + # XXX Sometimes we need to parenthesize arg or new. Later. + parent.parent.replace(parent, new) + +def diff(fn, tree): + f = open("@", "w") + try: + f.write(str(tree)) + finally: + f.close() + try: + return os.system("diff -u %s @" % fn) + finally: + os.remove("@") + +if __name__ == "__main__": + main() Modified: sandbox/trunk/2to3/pgen2/driver.py ============================================================================== --- sandbox/trunk/2to3/pgen2/driver.py (original) +++ sandbox/trunk/2to3/pgen2/driver.py Thu Nov 30 07:15:54 2006 @@ -22,7 +22,6 @@ # Pgen imports from pgen2 import parse -from pgen2 import astnode from pgen2 import grammar class Driver(object): Modified: sandbox/trunk/2to3/pgen2/parse.py ============================================================================== --- sandbox/trunk/2to3/pgen2/parse.py (original) +++ sandbox/trunk/2to3/pgen2/parse.py Thu Nov 30 07:15:54 2006 @@ -80,9 +80,7 @@ symbols, and None for tokens. An abstract syntax tree node may be anything; this is entirely - up to the converter function. For example, it can be an - instance of a subclass of the astnode.Node class (see the - astnode module). + up to the converter function. """ self.grammar = grammar Modified: sandbox/trunk/2to3/pgen2/test.py ============================================================================== --- sandbox/trunk/2to3/pgen2/test.py (original) +++ sandbox/trunk/2to3/pgen2/test.py Thu Nov 30 07:15:54 2006 @@ -1,6 +1,9 @@ # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. +# Modifications: +# Copyright 2006 Python Software Foundation. All Rights Reserved. + def test(): import sys sys.path[0] = ".." @@ -11,7 +14,8 @@ finally: f.close() sample = "year<=1989 ? ('Modula-3' + ABC) ** 2 : Python" - tree = driver.parse_string(sample, True) + dr = driver.Driver(driver.load_grammar()) + tree = dr.parse_string(sample, True) print tree if __name__ == "__main__": Modified: sandbox/trunk/2to3/play.py ============================================================================== --- sandbox/trunk/2to3/play.py (original) +++ sandbox/trunk/2to3/play.py Thu Nov 30 07:15:54 2006 @@ -1,9 +1,7 @@ #!/usr/bin/env python2.5 # Copyright 2006 Python Software Foundation. All Rights Reserved. -"""XXX.""" - -##from __future__ import with_statement +"""Main program for testing the infrastructure.""" __author__ = "Guido van Rossum <guido(a)python.org>" @@ -19,22 +17,17 @@ logging.basicConfig(level=logging.WARN) -def diff(fn, tree): - f = open("@", "w") - try: - f.write(str(tree)) - finally: - f.close() - return os.system("diff -u %s @" % fn) - def main(): gr = driver.load_grammar("Grammar.txt") dr = driver.Driver(gr, convert=pynode.convert) tree = dr.parse_file("example.py", debug=True) + tree.set_parents() sys.stdout.write(str(tree)) return # Comment out to run the complete test suite below + problems = [] + # Process every imported module for name in sys.modules: mod = sys.modules[name] @@ -47,7 +40,8 @@ continue print >>sys.stderr, "Parsing", fn tree = dr.parse_file(fn, debug=True) - diff(fn, tree) + if diff(fn, tree): + problems.append(fn) # Process every single module on sys.path (but not in packages) for dir in sys.path: @@ -66,7 +60,27 @@ except pgen2.parse.ParseError, err: print "ParseError:", err else: - diff(fn, tree) + if diff(fn, tree): + problems.append(fn) + + # Show summary of problem files + if not problems: + print "No problems. Congratulations!" + else: + print "Problems in following files:" + for fn in problems: + print "***", fn + +def diff(fn, tree): + f = open("@", "w") + try: + f.write(str(tree)) + finally: + f.close() + try: + return os.system("diff -u %s @" % fn) + finally: + os.remove("@") if __name__ == "__main__": main() Modified: sandbox/trunk/2to3/pynode.py ============================================================================== --- sandbox/trunk/2to3/pynode.py (original) +++ sandbox/trunk/2to3/pynode.py Thu Nov 30 07:15:54 2006 @@ -29,15 +29,17 @@ logger = logging.getLogger() class Node(object): + + # XXX Should refactor this so that there are only two kinds of nodes, + # Terminal and Nonterminal; each with subclasses to match the grammar + # or perhaps just storing the node type in a slot. + """Abstract base class for all nodes. - This has no attributes except a context slot which holds the line - number (or more detailed context info). In the future this might - change this to several slots (e.g. filename, lineno, column, or - even filename, start_lineno, start_column, end_lineno, - end_column). The context is only referenced by two places: the - part of the code that sticks it in, and the part of the code that - reports errors. + This has no attributes except a context slot which holds context + info (a tuple of the form (prefix, (lineno, column))), and a + parent slot, which is not set by default but can be set to the + parent node later. In order to reduce the amount of boilerplate code, the context is argument is handled by __new__ rather than __init__. There are @@ -46,7 +48,7 @@ """ - __slots__ = ["context"] + __slots__ = ["context", "parent"] def __new__(cls, context, *rest): assert cls not in (Node, Nonterminal, Terminal, Constant) @@ -54,6 +56,14 @@ obj.context = context return obj + def get_children(self): + return () + + def set_parents(self, parent=None): + self.parent = parent + for child in self.get_children(): + child.set_parents(self) + _stretch = False # Set to true to stretch the repr() vertically def __repr__(self, repr_arg=repr): @@ -101,6 +111,28 @@ def __str__(self): return self.__repr__(repr_arg=str) + def __eq__(self, other): + if self.__class__ is not other.__class__: + return NotImplemented + return self.eq(other) + + def __ne__(self, other): + result = self.__eq__(other) + if result is not NotImplemented: + result = not result + return result + + def eq(self, other): + assert self.__class__ is other.__class__ + return self.get_children() == other.get_children() + + def set_prefix(self, new_prefix): + old_prefix, rest = self.context + self.context = (new_prefix, rest) + + def get_prefix(self): + return self.context[0] + class Nonterminal(Node): """Abstract base class for nonterminal symbols. @@ -134,7 +166,7 @@ return nodes[0] else: obj = Nonterminal.__new__(cls, context) - obj.initseries(nodes) + obj.init_series(nodes) return obj class Constant(Terminal): @@ -152,9 +184,12 @@ self.repr = repr def __str__(self): - prefix, start = self.context + prefix, (lineno, column) = self.context return prefix + self.repr + def eq(self, other): + return self.repr == other.repr + # Node classes for terminal symbols class Token(Constant): @@ -186,6 +221,9 @@ prefix, start = self.context return prefix + self.name + def eq(self, other): + return self.name == other.name + class Number(Constant): """Numeric constant. @@ -207,12 +245,28 @@ # Nodes and factory functions for Python grammar class GenericSeries(Series): + __slots__ = ["nodes"] - def initseries(self, nodes): + + def init_series(self, nodes): self.nodes = nodes + + def get_children(self): + return self.nodes + def __str__(self): return "".join(map(str, self.nodes)) + def replace(self, old, new): + self.nodes = tuple((new if n is old else n) for n in self.nodes) + + def set_prefix(self, new_prefix): + Series.set_prefix(self, new_prefix) + self.nodes[0].set_prefix(new_prefix) + + def get_prefix(self): + return self.nodes[0].get_prefix() + class atom(GenericSeries): __slots__ = []

1 0

r52867 - sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 04:29:39 2006 New Revision: 52867 Modified: sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py Log: Lose lots of unused code from pynode.py. Modified: sandbox/trunk/2to3/play.py ============================================================================== --- sandbox/trunk/2to3/play.py (original) +++ sandbox/trunk/2to3/play.py Thu Nov 30 04:29:39 2006 @@ -33,7 +33,7 @@ tree = dr.parse_file("example.py", debug=True) sys.stdout.write(str(tree)) - return + return # Comment out to run the complete test suite below # Process every imported module for name in sys.modules: Modified: sandbox/trunk/2to3/pynode.py ============================================================================== --- sandbox/trunk/2to3/pynode.py (original) +++ sandbox/trunk/2to3/pynode.py Thu Nov 30 04:29:39 2006 @@ -9,11 +9,6 @@ There is a class or function corresponding to each terminal and nonterminal symbol. -The grammar is pieced together from the docstrings of the -corresponding classes and functions: text between dollar signs is -assumed to be the right-hand side of the grammar rule corresponding to -that class or function. - We use __slots__ to make the parse tree nodes as small as possible. NOTE: EVERY CLASS MUST HAVE A __slots__ DEFINITION, EVEN IF EMPTY! @@ -211,11 +206,6 @@ # Nodes and factory functions for Python grammar -class VanishingNonterminal(Nonterminal): - __slots__ = [] - def __new__(cls, context, node): - return node - class GenericSeries(Series): __slots__ = ["nodes"] def initseries(self, nodes): @@ -466,182 +456,6 @@ class file_input(GenericSeries): __slots__ = [] - -# Example nodes and factory functions for nonterminal symbols -# XXX: get rid of these - -def Program(context, stmts): - """Program is a nonterminal with only one non-trivial child. - - Grammar: $ Expression ENDMARKER $ - - """ - return stmts - -class Expression(Series): - "Grammar: $ BinaryExpression ['?' BinaryExpression ':' BinaryExpression] $" - - __slots__ = ["test", "left", "right"] - - def initseries(self, nodes): - self.test, self.left, self.right = nodes - - def __str__(self): - return "%s ? %s : %s" % (self.test, self.left, self.right) - -class BinaryExpression(Series): - "Grammar: $ Operand (Operator Operand)* $" - - __slots__ = ["left", "op", "right"] - - def initseries(self, stuff): - # Stuff is a list with alternating operands and operators - if len(stuff) == 3: - self.left, self.op, self.right = stuff - return - assert len(stuff) > 1 and len(stuff) % 2 == 1 - # Find the rightmost lowest-priority operator - lowest_i = 1 - lowest_op = stuff[lowest_i] - lowest_pri = lowest_op.priority() - for i in range(3, len(stuff), 2): - op = stuff[i] - pri = op.priority() - if pri <= lowest_pri: - lowest_i = i - lowest_op = op - lowest_pri = pri - self.left = self.__class__(self.context, *stuff[:lowest_i]) - self.op = lowest_op - self.right = self.__class__(self.context, *stuff[lowest_i+1:]) - - def __str__(self): - return "(%s %s %s)" % (self.left, self.op, self.right) - -def Operand(context, arg): - """Operand is a nonterminal with one child. - - Grammar: $ Atom | UnaryExpression $ - - """ - return arg - -class UnaryExpression(Nonterminal): - "Grammar: $ UnaryOperator Operand $" - - __slots__ = ["op", "arg"] - - def __init__(self, context, op, arg): - self.op = op - self.arg = arg - - def __str__(self): - return "%s%s" % (self.op, self.arg) - -def UnaryOperator(context, op): - "Grammar: $ 'not' | '-' $" - return op - -class XXXOperator(Nonterminal): - """Operator. - - This has a repr slot and a priority() method. - - The __new__ method implements a sort-of singleton pattern: there's - only one instance per repr value. (Yes, this means the context is - not useful. Therefore we set it to None.) - - Grammar: $ '**' | '*' | '/' | '+' | '-' | '&' | '^' | '|' | - ['not'] 'in' | '==' | '<' | '>' | '!=' | '<=' | '>=' | - 'and' | 'or' $ - - """ - - __slots__ = ["repr"] - - _stretch = False - - _cache = {} - - def __new__(cls, context, *args): - repr = " ".join(args) - # For "not in", the argument should be the string "not in" - obj = cls._cache.get(repr) - if obj is None: - obj = Terminal.__new__(cls, None) - obj.repr = repr - return obj - - def __str__(self): - return self.repr - - def priority(self): - return self._priorities[self.repr] - - _priorities = { - "or": 0, - "and": 1, - "in": 2, - "not in": 2, - "<": 2, - ">": 2, - "==": 2, - "<=": 2, - ">=": 2, - "!=": 2, - "|": 3, - "^": 4, - "&": 5, - "+": 6, - "-": 6, - "*": 7, - "/": 7, - "**": 8, - } - -def Atom(context, arg): - """ Grammar: $ NAME | STRING | NUMBER | ParenthesizedExpression $ - """ - return arg - -def ParenthesizedExpression(context, expr): - "Grammar: $ '(' Expression ')' $" - return expr - - -# Conversion from concrete to abstract syntax trees - -def vanish(context, value): - return None - -token_mapping = { - # Tokens that become parse tree nodes - # (token.NAME is special-cased in the code) - token.NUMBER: Number, - token.STRING: String, - -## # Tokens that vanish -## token.DOT: vanish, -## token.LPAR: vanish, -## token.RPAR: vanish, -## token.COLON: vanish, -## token.COMMA: vanish, -## token.EQUAL: vanish, -## token.DEDENT: vanish, -## token.INDENT: vanish, -## token.LBRACE: vanish, -## token.RBRACE: vanish, -## token.NEWLINE: vanish, -## token.ENDMARKER: vanish, -## grammar.QUESTIONMARK: vanish, - - # All other tokens return the token's string value (e.g. "+") - } - -vanishing_keywords = { - # E.g. "def": True, - } - def convert(grammar, node): type, value, context, children = node # Is it a non-terminal symbol? @@ -667,47 +481,12 @@ if type == token.NAME: # Name or keyword. Special-case the snot out of this. if value in grammar.keywords: - # Keywords become strings, like operators - if value in vanishing_keywords: - return None - else: - return Token(context, value) + # Keywords become Tokens + return Token(context, value) else: return Name(context, value) - # Look for a handler in the token_mapping table. assert type in token.tok_name - factory = token_mapping.get(type) - if factory: - return factory(context, value) - else: - return Token(context, value) - - -# Support code - -def generate_grammar(stream): - """Extract the grammar rules from this module's doc strings.""" - import re - from types import ModuleType - lines = [] - startline = None - for name, obj in globals().items(): - if hasattr(obj, "__doc__") and not isinstance(obj, ModuleType): - m = re.search(r"Grammar:\s*\$([^$]+)\$", obj.__doc__ or "") - if m: - rule = obj.__name__, " ".join(m.group(1).split()) - if rule[0] == "Program": - assert not startline - startline = rule - else: - lines.append(rule) - lines.sort() - # The start symbol *must* be the first rule - lines.insert(0, startline) - for name, rhs in lines: - stream.write("%s: %s\n" % (name, rhs)) - -if __name__ == '__main__': - import sys - generate_grammar(sys.stdout) + + # Operators become Tokens + return Token(context, value)

1 0

r52866 - in sandbox/trunk/2to3: example.py pgen2/driver.py play.py pynode.py tokenize.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 02:48:54 2006 New Revision: 52866 Modified: sandbox/trunk/2to3/example.py sandbox/trunk/2to3/pgen2/driver.py sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py sandbox/trunk/2to3/tokenize.py Log: We now correctly render all of the standard library's top-level modules, except for files containing ^L characters. Modified: sandbox/trunk/2to3/example.py ============================================================================== --- sandbox/trunk/2to3/example.py (original) +++ sandbox/trunk/2to3/example.py Thu Nov 30 02:48:54 2006 @@ -1 +1,8 @@ -print (12 + 12) +#!/usr/bin/python +"""Docstring. +""" +print (12 + # Hello + # world + + 12) +# This is the last line. Modified: sandbox/trunk/2to3/pgen2/driver.py ============================================================================== --- sandbox/trunk/2to3/pgen2/driver.py (original) +++ sandbox/trunk/2to3/pgen2/driver.py Thu Nov 30 02:48:54 2006 @@ -38,19 +38,42 @@ """Parse a stream and return the concrete syntax tree.""" p = parse.Parser(self.grammar, self.convert) p.setup() - t = v = x = None - # (t, v, x, y, z) == (type, value, start, end, line) - for t, v, x, y, z in tokenize.generate_tokens(stream.readline): - if t in (tokenize.COMMENT, tokenize.NL): + lineno = 1 + column = 0 + type = value = start = end = line_text = None + prefix = "" + for quintuple in tokenize.generate_tokens(stream.readline): + type, value, start, end, line_text = quintuple + if start != (lineno, column): + assert (lineno, column) <= start, ((lineno, column), start) + s_lineno, s_column = start + if lineno < s_lineno: + prefix += "\n" * (s_lineno - lineno) + lineno = s_lineno + column = 0 + if column < s_column: + prefix += " " * (s_column - column) + column = s_column + if type in (tokenize.COMMENT, tokenize.NL): + prefix += value + lineno, column = end + if value.endswith("\n"): + lineno += 1 + column = 0 continue - if t == token.OP: - t = grammar.opmap[v] + if type == token.OP: + type = grammar.opmap[value] if debug: - self.logger.debug("%s %r", token.tok_name[t], v) - if p.addtoken(t, v, x): + self.logger.debug("%s %r", token.tok_name[type], value) + if p.addtoken(type, value, (prefix, start)): if debug: self.logger.debug("Stop.") break + prefix = "" + lineno, column = end + if value.endswith("\n"): + lineno += 1 + column = 0 else: # We never broke out -- EOF is too soon (how can this happen???) raise parse.ParseError("incomplete input", t, v, x) Modified: sandbox/trunk/2to3/play.py ============================================================================== --- sandbox/trunk/2to3/play.py (original) +++ sandbox/trunk/2to3/play.py Thu Nov 30 02:48:54 2006 @@ -19,41 +19,54 @@ logging.basicConfig(level=logging.WARN) +def diff(fn, tree): + f = open("@", "w") + try: + f.write(str(tree)) + finally: + f.close() + return os.system("diff -u %s @" % fn) + def main(): gr = driver.load_grammar("Grammar.txt") dr = driver.Driver(gr, convert=pynode.convert) tree = dr.parse_file("example.py", debug=True) - print tree + sys.stdout.write(str(tree)) + return -## # Process every imported module -## for name in sys.modules: -## mod = sys.modules[name] -## if mod is None or not hasattr(mod, "__file__"): -## continue -## fn = mod.__file__ -## if fn.endswith(".pyc"): -## fn = fn[:-1] -## if not fn.endswith(".py"): -## continue -## print >>sys.stderr, "Parsing", fn -## dr.parse_file(fn, debug=True) - -## # Process every single module on sys.path (but not in packages) -## for dir in sys.path: -## try: -## names = os.listdir(dir) -## except os.error: -## continue -## print >>sys.stderr, "Scanning", dir, "..." -## for name in names: -## if not name.endswith(".py"): -## continue -## print >>sys.stderr, "Parsing", name -## try: -## dr.parse_file(os.path.join(dir, name), debug=True) -## except pgen2.parse.ParseError, err: -## print "ParseError:", err + # Process every imported module + for name in sys.modules: + mod = sys.modules[name] + if mod is None or not hasattr(mod, "__file__"): + continue + fn = mod.__file__ + if fn.endswith(".pyc"): + fn = fn[:-1] + if not fn.endswith(".py"): + continue + print >>sys.stderr, "Parsing", fn + tree = dr.parse_file(fn, debug=True) + diff(fn, tree) + + # Process every single module on sys.path (but not in packages) + for dir in sys.path: + try: + names = os.listdir(dir) + except os.error: + continue + print >>sys.stderr, "Scanning", dir, "..." + for name in names: + if not name.endswith(".py"): + continue + print >>sys.stderr, "Parsing", name + fn = os.path.join(dir, name) + try: + tree = dr.parse_file(fn, debug=True) + except pgen2.parse.ParseError, err: + print "ParseError:", err + else: + diff(fn, tree) if __name__ == "__main__": main() Modified: sandbox/trunk/2to3/pynode.py ============================================================================== --- sandbox/trunk/2to3/pynode.py (original) +++ sandbox/trunk/2to3/pynode.py Thu Nov 30 02:48:54 2006 @@ -61,7 +61,7 @@ _stretch = False # Set to true to stretch the repr() vertically - def __repr__(self, repr=repr): + def __repr__(self, repr_arg=repr): stretch = self._stretch r = [self.__class__.__name__] if stretch: @@ -93,10 +93,10 @@ except AttributeError: continue if stretch and isinstance(value, list): - rr = map(repr, value) + rr = map(repr_arg, value) rv = "[" + ",\n ".join(rr) + "]" else: - rv = repr(value) + rv = repr_arg(value) if stretch: rv = rv.replace("\n", "\n ") r.append(rv) @@ -104,7 +104,7 @@ return "".join(r) def __str__(self): - return self.__repr__(repr=str) + return self.__repr__(repr_arg=str) class Nonterminal(Node): """Abstract base class for nonterminal symbols. @@ -157,10 +157,22 @@ self.repr = repr def __str__(self): - return self.repr + prefix, start = self.context + return prefix + self.repr # Node classes for terminal symbols +class Token(Constant): + """An otherwise unclassified operator or keyword (e.g. '+' or 'if'). + + Attributres: + + repr -- a string giving the token's text. + + """ + + __slots__ = [] + class Name(Terminal): """Name (e.g. a variable name or an attribute name). @@ -176,7 +188,8 @@ self.name = name def __str__(self): - return self.name + prefix, start = self.context + return prefix + self.name class Number(Constant): """Numeric constant. @@ -208,7 +221,7 @@ def initseries(self, nodes): self.nodes = nodes def __str__(self): - return " ".join(map(str, self.nodes)) + return "".join(map(str, self.nodes)) class atom(GenericSeries): __slots__ = [] @@ -529,7 +542,7 @@ "Grammar: $ 'not' | '-' $" return op -class Operator(Nonterminal): +class XXXOperator(Nonterminal): """Operator. This has a repr slot and a priority() method. @@ -619,7 +632,7 @@ ## token.LBRACE: vanish, ## token.RBRACE: vanish, ## token.NEWLINE: vanish, - token.ENDMARKER: vanish, +## token.ENDMARKER: vanish, ## grammar.QUESTIONMARK: vanish, # All other tokens return the token's string value (e.g. "+") @@ -658,7 +671,7 @@ if value in vanishing_keywords: return None else: - return value + return Token(context, value) else: return Name(context, value) @@ -668,7 +681,7 @@ if factory: return factory(context, value) else: - return value + return Token(context, value) # Support code Modified: sandbox/trunk/2to3/tokenize.py ============================================================================== --- sandbox/trunk/2to3/tokenize.py (original) +++ sandbox/trunk/2to3/tokenize.py Thu Nov 30 02:48:54 2006 @@ -370,6 +370,8 @@ elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt + # This yield is new; needed for better idempotency: + yield (NL, initial, spos, (spos[0], spos[1]+1), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1

1 0

r52865 - sandbox/trunk/2to3/tokenize.py
by guido.van.rossum 30 Nov '06

30 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 02:16:57 2006 New Revision: 52865 Added: sandbox/trunk/2to3/tokenize.py - copied unchanged from r52864, python/trunk/Lib/tokenize.py Log: A copy of tokenize.py from the trunk.

1 0

buildbot warnings in x86 Ubuntu edgy (icc) 2.5
by buildbot＠python.org 29 Nov '06

29 Nov '06

The Buildbot has detected a new failure of x86 Ubuntu edgy (icc) 2.5. Full details are available at: http://www.python.org/dev/buildbot/all/x86%2520Ubuntu%2520edgy%2520%2528icc… Buildbot URL: http://www.python.org/dev/buildbot/all/ Build Reason: Build Source Stamp: [branch branches/release25-maint] HEAD Blamelist: armin.rigo Build had warnings: warnings test Excerpt from the test logfile: 1 test failed: test_timeout ====================================================================== FAIL: testConnectTimeout (test.test_timeout.TimeoutTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/buildbot/Buildbot/2.5.baxter-ubuntu/build/Lib/test/test_timeout.py", line 128, in testConnectTimeout %(_delta, self.fuzz, _timeout)) AssertionError: timeout (3.04095) is more than 2 seconds more than expected (0.001) sincerely, -The Buildbot

1 0

r52864 - sandbox/trunk/2to3/example.py sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py
by guido.van.rossum 29 Nov '06

29 Nov '06

Author: guido.van.rossum Date: Thu Nov 30 00:19:02 2006 New Revision: 52864 Added: sandbox/trunk/2to3/example.py (contents, props changed) Modified: sandbox/trunk/2to3/play.py sandbox/trunk/2to3/pynode.py Log: Add specific example code. Run the example by default. Add __str__ to generic series node. Don't require Python 2.5. Added: sandbox/trunk/2to3/example.py ============================================================================== --- (empty file) +++ sandbox/trunk/2to3/example.py Thu Nov 30 00:19:02 2006 @@ -0,0 +1 @@ +print (12 + 12) Modified: sandbox/trunk/2to3/play.py ============================================================================== --- sandbox/trunk/2to3/play.py (original) +++ sandbox/trunk/2to3/play.py Thu Nov 30 00:19:02 2006 @@ -3,7 +3,7 @@ """XXX.""" -from __future__ import with_statement +##from __future__ import with_statement __author__ = "Guido van Rossum <guido(a)python.org>" @@ -17,23 +17,16 @@ import pynode -logging.basicConfig(level=logging.INFO) - -SAMPLE = """\ -for i in range(10): - print i -class C(object, list): - def f(a, b, c=1, *args, **kwds): - pass -""" +logging.basicConfig(level=logging.WARN) def main(): gr = driver.load_grammar("Grammar.txt") dr = driver.Driver(gr, convert=pynode.convert) -## tree = dr.parse_file("pynode.py", debug=True) -## print tree + tree = dr.parse_file("example.py", debug=True) + print tree +## # Process every imported module ## for name in sys.modules: ## mod = sys.modules[name] ## if mod is None or not hasattr(mod, "__file__"): @@ -46,21 +39,21 @@ ## print >>sys.stderr, "Parsing", fn ## dr.parse_file(fn, debug=True) - for dir in reversed(sys.path): - try: - names = os.listdir(dir) - except os.error: - continue - print >>sys.stderr, "Scanning", dir, "..." - for name in names: - if not name.endswith(".py"): - continue - print >>sys.stderr, "Parsing", name - try: - dr.parse_file(os.path.join(dir, name), debug=True) - except pgen2.parse.ParseError, err: - print "ParseError:", err - +## # Process every single module on sys.path (but not in packages) +## for dir in sys.path: +## try: +## names = os.listdir(dir) +## except os.error: +## continue +## print >>sys.stderr, "Scanning", dir, "..." +## for name in names: +## if not name.endswith(".py"): +## continue +## print >>sys.stderr, "Parsing", name +## try: +## dr.parse_file(os.path.join(dir, name), debug=True) +## except pgen2.parse.ParseError, err: +## print "ParseError:", err if __name__ == "__main__": main() Modified: sandbox/trunk/2to3/pynode.py ============================================================================== --- sandbox/trunk/2to3/pynode.py (original) +++ sandbox/trunk/2to3/pynode.py Thu Nov 30 00:19:02 2006 @@ -207,6 +207,8 @@ __slots__ = ["nodes"] def initseries(self, nodes): self.nodes = nodes + def __str__(self): + return " ".join(map(str, self.nodes)) class atom(GenericSeries): __slots__ = []

1 0

buildbot failure in x86 Ubuntu edgy (icc) trunk
by buildbot＠python.org 29 Nov '06

29 Nov '06

The Buildbot has detected a new failure of x86 Ubuntu edgy (icc) trunk. Full details are available at: http://www.python.org/dev/buildbot/all/x86%2520Ubuntu%2520edgy%2520%2528icc… Buildbot URL: http://www.python.org/dev/buildbot/all/ Build Reason: Build Source Stamp: [branch trunk] HEAD Blamelist: armin.rigo BUILD FAILED: failed failed slave lost sincerely, -The Buildbot

1 0