[Python-checkins] r53006 - in sandbox/trunk/2to3: fix_apply.py fixes fixes/__init__.py fixes/fix_apply.py fixes/fix_null.py pygram.py pytree.py refactor.py
guido.van.rossum
python-checkins at python.org
Tue Dec 12 15:56:30 CET 2006
Author: guido.van.rossum
Date: Tue Dec 12 15:56:29 2006
New Revision: 53006
Added:
sandbox/trunk/2to3/fixes/ (props changed)
sandbox/trunk/2to3/fixes/__init__.py (contents, props changed)
sandbox/trunk/2to3/fixes/fix_apply.py (contents, props changed)
sandbox/trunk/2to3/fixes/fix_null.py (contents, props changed)
sandbox/trunk/2to3/pygram.py (contents, props changed)
Removed:
sandbox/trunk/2to3/fix_apply.py
Modified:
sandbox/trunk/2to3/pytree.py
sandbox/trunk/2to3/refactor.py
Log:
Refactored the refactoring -- fixes now live in subpackage "fixes"
and are defined by a class with a specific API. Not all fixes
have been converted yet.
Deleted: /sandbox/trunk/2to3/fix_apply.py
==============================================================================
--- /sandbox/trunk/2to3/fix_apply.py Tue Dec 12 15:56:29 2006
+++ (empty file)
@@ -1,137 +0,0 @@
-#!/usr/bin/env python2.5
-# Copyright 2006 Google Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""Refactoring tool: change apply(f, a, kw) into f(*a, **kw)."""
-
-__author__ = "Guido van Rossum <guido at python.org>"
-
-# Python imports
-import os
-import sys
-import token
-import logging
-
-import pgen2
-from pgen2 import driver
-
-import pytree
-import patcomp
-
-logging.basicConfig(level=logging.DEBUG)
-
-gr = driver.load_grammar("Grammar.txt") # used by node initializers
-
-
-class Symbols(object):
-
- def __init__(self, gr):
- for name, symbol in gr.symbol2number.iteritems():
- setattr(self, name, symbol)
-
-
-# XXX Make this importable as a module
-syms = Symbols(gr)
-
-
-# XXX Write a single driver script that can do any (or all) refactorings
-def main():
- args = sys.argv[1:] or ["example.py"]
-
- dr = driver.Driver(gr, convert=pytree.convert)
-
- for fn in args:
- print "Parsing", fn
- tree = dr.parse_file(fn)
- refactor(tree)
- diff(fn, tree)
-
-
-def refactor(tree):
- visit(tree, fix_apply)
-
-
-def visit(node, func):
- func(node)
- for child in node.children:
- visit(child, func)
-
-
-# Constant nodes used for matching
-n_comma = pytree.Leaf(token.COMMA, ",")
-n_star = pytree.Leaf(token.STAR, "*")
-n_doublestar = pytree.Leaf(token.DOUBLESTAR, "**")
-
-# Tree matching patterns
-pat_compile = patcomp.PatternCompiler().compile_pattern
-p_apply = pat_compile("""
-power< 'apply'
- trailer<
- '('
- arglist<
- (not argument<NAME '=' any>) func=any ','
- (not argument<NAME '=' any>) args=any [','
- (not argument<NAME '=' any>) kwds=any] [',']
- >
- ')'
- >
->
-"""
- )
-
-
-def fix_apply(node):
- results = {}
- if not p_apply.match(node, results):
- return
- n_arglist = node.children[1].children[1]
- assert n_arglist.type
- func = results["func"]
- args = results["args"]
- kwds = results.get("kwds")
- prefix = node.get_prefix()
- func.replace(None)
- if (func.type not in (token.NAME, syms.atom) and
- (func.type != syms.power or
- func.children[-2].type == token.DOUBLESTAR)):
- # Need to parenthesize
- func = pytree.Node(syms.atom,
- (pytree.Leaf(token.LPAR, "("),
- func,
- pytree.Leaf(token.RPAR, ")")))
- func.set_prefix("")
- args.replace(None)
- args.set_prefix("")
- if kwds is not None:
- kwds.replace(None)
- kwds.set_prefix("")
- node.children[0].replace(func)
- node.set_prefix(prefix)
- l_newargs = [pytree.Leaf(token.STAR, "*"), args]
- if kwds is not None:
- l_newargs.extend([pytree.Leaf(token.COMMA, ","),
- pytree.Leaf(token.DOUBLESTAR, "**"),
- kwds])
- l_newargs[-2].set_prefix(" ") # that's the ** token
- for n in l_newargs:
- if n.parent is not None:
- n.replace(None) # Force parent to None
- n_arglist.replace(pytree.Node(syms.arglist, l_newargs))
- # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
- # can be translated into f(x, y, *t) instead of f(*(x, y) + t)
-
-
-def diff(fn, tree):
- f = open("@", "w")
- try:
- f.write(str(tree))
- finally:
- f.close()
- try:
- return os.system("diff -u %s @" % fn)
- finally:
- os.remove("@")
-
-
-if __name__ == "__main__":
- main()
Added: sandbox/trunk/2to3/fixes/__init__.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/fixes/__init__.py Tue Dec 12 15:56:29 2006
@@ -0,0 +1 @@
+# Dummy file to make this directory a package.
Added: sandbox/trunk/2to3/fixes/fix_apply.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/fixes/fix_apply.py Tue Dec 12 15:56:29 2006
@@ -0,0 +1,81 @@
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Fixer for apply()."""
+
+# Python imports
+import token
+
+# Local imports
+import pytree
+import patcomp
+from pygram import python_symbols as syms
+
+pat_compile = patcomp.PatternCompiler().compile_pattern
+
+PATTERN = """
+power< 'apply'
+ trailer<
+ '('
+ arglist<
+ (not argument<NAME '=' any>) func=any ','
+ (not argument<NAME '=' any>) args=any [','
+ (not argument<NAME '=' any>) kwds=any] [',']
+ >
+ ')'
+ >
+>
+"""
+
+
+class FixApply(object):
+
+ def __init__(self, options):
+ self.options = options
+ self.pattern = pat_compile(PATTERN)
+
+ def match(self, node):
+ results = {}
+ return self.pattern.match(node, results) and results
+
+ def transform(self, node):
+ results = self.match(node)
+ assert results
+ if not results:
+ return
+ assert node.children[1].children[1].type == syms.arglist
+ func = results["func"]
+ args = results["args"]
+ kwds = results.get("kwds")
+ prefix = node.get_prefix()
+ func = func.clone()
+ if (func.type not in (token.NAME, syms.atom) and
+ (func.type != syms.power or
+ func.children[-2].type == token.DOUBLESTAR)):
+ # Need to parenthesize
+ func = pytree.Node(syms.atom,
+ (pytree.Leaf(token.LPAR, "("),
+ func,
+ pytree.Leaf(token.RPAR, ")")))
+ func.set_prefix("")
+ args = args.clone()
+ args.set_prefix("")
+ if kwds is not None:
+ kwds = kwds.clone()
+ kwds.set_prefix("")
+ l_newargs = [pytree.Leaf(token.STAR, "*"), args]
+ if kwds is not None:
+ l_newargs.extend([pytree.Leaf(token.COMMA, ","),
+ pytree.Leaf(token.DOUBLESTAR, "**"),
+ kwds])
+ l_newargs[-2].set_prefix(" ") # that's the ** token
+ # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t)
+ # can be translated into f(x, y, *t) instead of f(*(x, y) + t)
+ new = pytree.Node(syms.power,
+ (func,
+ pytree.Node(syms.trailer,
+ (pytree.Leaf(token.LPAR, "("),
+ pytree.Node(syms.arglist, l_newargs),
+ pytree.Leaf(token.RPAR, ")")))))
+ new.set_prefix(prefix)
+ return new
Added: sandbox/trunk/2to3/fixes/fix_null.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/fixes/fix_null.py Tue Dec 12 15:56:29 2006
@@ -0,0 +1,40 @@
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Null fixer. Use as a template."""
+
+
+class FixNull(object):
+
+ """Fixer class.
+
+ The class name must be FixFooBar where FooBar is the result of
+ removing underscores and capitalizing the words of the fix name.
+ For example, the class name for a fixer named 'has_key' should be
+ FixHasKey.
+ """
+
+ def __init__(self, options):
+ """Initializer.
+
+ The argument is an optparse.Values instance which can be used
+ to inspect the command line options.
+ """
+ self.options = options
+
+ def match(self, node):
+ """Matcher.
+
+ Should return a true or false object (not necessarily a bool).
+ It may return a non-empty dict of matching sub-nodes as
+ returned by a matching pattern.
+ """
+ return None
+
+ def transform(self, node):
+ """Transformer.
+
+ Should return None, or a node that is a modified copy of the
+ argument node. The argument should not be modified in place.
+ """
+ return None
Added: sandbox/trunk/2to3/pygram.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/2to3/pygram.py Tue Dec 12 15:56:29 2006
@@ -0,0 +1,17 @@
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Export the Python grammar and symbols."""
+
+from pgen2 import driver
+
+
+class Symbols(object):
+
+ def __init__(self, gr):
+ for name, symbol in gr.symbol2number.iteritems():
+ setattr(self, name, symbol)
+
+
+python_grammar = driver.load_grammar("Grammar.txt")
+python_symbols = Symbols(python_grammar)
Modified: sandbox/trunk/2to3/pytree.py
==============================================================================
--- sandbox/trunk/2to3/pytree.py (original)
+++ sandbox/trunk/2to3/pytree.py Tue Dec 12 15:56:29 2006
@@ -64,6 +64,13 @@
"""
raise NotImplementedError
+ def clone(self):
+ """Returns a cloned (deep) copy of self.
+
+ This must be implemented by the concrete subclass.
+ """
+ raise NotImplementedError
+
def post_order(self):
"""Returns a post-order iterator for the tree.
@@ -126,7 +133,7 @@
self.type = type
self.children = tuple(children)
for ch in self.children:
- assert ch.parent is None, str(ch)
+ assert ch.parent is None, repr(ch)
ch.parent = self
def __repr__(self):
@@ -146,6 +153,10 @@
"""Compares two nodes for equality."""
return (self.type, self.children) == (other.type, other.children)
+ def clone(self):
+ """Returns a cloned (deep) copy of self."""
+ return Node(self.type, (ch.clone() for ch in self.children))
+
def post_order(self):
"""Returns a post-order iterator for the tree."""
for child in self.children:
@@ -209,6 +220,11 @@
"""Compares two nodes for equality."""
return (self.type, self.value) == (other.type, other.value)
+ def clone(self):
+ """Returns a cloned (deep) copy of self."""
+ return Leaf(self.type, self.value,
+ (self.prefix, (self.lineno, self.column)))
+
def post_order(self):
"""Returns a post-order iterator for the tree."""
yield self
Modified: sandbox/trunk/2to3/refactor.py
==============================================================================
--- sandbox/trunk/2to3/refactor.py (original)
+++ sandbox/trunk/2to3/refactor.py Tue Dec 12 15:56:29 2006
@@ -22,6 +22,8 @@
import pytree
import patcomp
from pgen2 import driver
+import fixes
+import pygram
def main(args=None):
@@ -37,7 +39,7 @@
parser.add_option("-f", "--fix", action="append", default=[],
help="Each FIX specifies a transformation; default all")
parser.add_option("-l", "--list-fixes", action="store_true",
- help="List available transformations")
+ help="List available transformations (fixes/fix_*.py)")
parser.add_option("-v", "--verbose", action="store_true",
help="More verbose logging")
@@ -45,7 +47,7 @@
options, args = parser.parse_args(args)
if options.list_fixes:
print "Available transformations for the -f/--fix option:"
- for fixname in get_all_fixes():
+ for fixname in get_all_fix_names():
print fixname
if not args:
return 0
@@ -65,14 +67,14 @@
return int(bool(rt.errors))
-def get_all_fixes():
- """Return a sorted list of all available fixes."""
- fixes = []
- for name in os.listdir(os.path.dirname(__file__)):
+def get_all_fix_names():
+ """Return a sorted list of all available fix names."""
+ fix_names = []
+ for name in os.listdir(os.path.dirname(fixes.__file__)):
if name.startswith("fix_") and name.endswith(".py"):
- fixes.append(name[4:-3])
- fixes.sort()
- return fixes
+ fix_names.append(name[4:-3])
+ fix_names.sort()
+ return fix_names
class RefactoringTool(object):
@@ -84,35 +86,40 @@
"""
self.options = options
self.errors = 0
- self.gr = driver.load_grammar("Grammar.txt")
- self.dr = dr = driver.Driver(self.gr, convert=pytree.convert)
- self.pairs = self.get_refactoring_pairs()
+ self.driver = driver.Driver(pygram.python_grammar,
+ convert=pytree.convert)
+ self.fixers = self.get_fixers()
- def get_refactoring_pairs(self):
+ def get_fixers(self):
"""Inspects the options to load the requested patterns and handlers."""
- pairs = []
- fixes = self.options.fix
- if not fixes or "all" in fixes:
- fixes = get_all_fixes()
- for fixname in fixes:
+ fixers = []
+ fix_names = self.options.fix
+ if not fix_names or "all" in fix_names:
+ fixes = get_all_fix_names()
+ for fix_name in fix_names:
try:
- mod = __import__("fix_" + fixname)
- except (ImportError, AttributeError):
- self.log_error("Can't find transformation %s", fixname)
- else:
- name = "?"
- try:
- name = "p_" + fixname
- pattern = getattr(mod, name)
- name = "fix_" + fixname
- handler = getattr(mod, name)
- except AttributeError:
- self.log_error("Can't find fix_%s.%s", fixname, name)
- else:
- if self.options.verbose:
- self.log_message("adding transformation: %s", fixname)
- pairs.append((pattern, handler))
- return pairs
+ mod = __import__("fixes.fix_" + fix_name, {}, {}, ["*"])
+ except ImportError:
+ self.log_error("Can't find transformation %s", fix_name)
+ continue
+ parts = fix_name.split("_")
+ class_name = "Fix" + "".join(p.title() for p in parts)
+ try:
+ fix_class = getattr(mod, class_name)
+ except AttributeError:
+ self.log_error("Can't find fixes.fix_%s.%s",
+ fix_name, class_name)
+ continue
+ try:
+ fixer = fix_class(self.options)
+ except Exception, err:
+ self.log_error("Can't instantiate fixes.fix_%s.%s()",
+ fix_name, class_name)
+ continue
+ if self.options.verbose:
+ self.log_message("Adding transformation: %s", fix_name)
+ fixers.append(fixer)
+ return fixers
def log_error(self, msg, *args):
"""Increment error count and log a message."""
@@ -159,7 +166,7 @@
return
try:
try:
- tree = self.dr.parse_file(filename)
+ tree = self.driver.parse_file(filename)
except Exception, err:
self.log_error("Can't parse %s: %s: %s",
filename, err.__class__.__name__, err)
@@ -174,11 +181,12 @@
def refactor_tree(self, tree):
changes = 0
for node in tree.post_order():
- for pattern, handler in self.pairs:
- if pattern.match(node):
- # XXX Change handler API to return a replacement node
- handler(node)
- changes += 1
+ for fixer in self.fixers:
+ if fixer.match(node):
+ new = fixer.transform(node)
+ if new is not None and new != node:
+ node.replace(new)
+ changes += 1
return changes
def save_tree(self, tree, filename):
@@ -202,7 +210,7 @@
# XXX Actually save it
pass
else:
- self.log_error("diff %s returned exit (%s,%s)",
+ self.log_error("Diff %s returned exit (%s,%s)",
filename, sts>>8, sts&0xFF)
finally:
os.remove(tfn)
More information about the Python-checkins
mailing list