[Python-checkins] r76517 - in python/trunk/Lib/lib2to3: fixes/fix_imports.py fixes/fix_next.py fixes/fix_renames.py fixes/fix_urllib.py main.py pgen2/tokenize.py refactor.py tests/data/bom.py tests/test_fixers.py tests/test_refactor.py
benjamin.peterson
python-checkins at python.org
Wed Nov 25 19:16:46 CET 2009
Author: benjamin.peterson
Date: Wed Nov 25 19:16:46 2009
New Revision: 76517
Log:
Merged revisions 76160-76161,76250,76252,76447,76506 via svnmerge from
svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3
........
r76160 | benjamin.peterson | 2009-11-08 18:53:48 -0600 (Sun, 08 Nov 2009) | 1 line
undeprecate the -p option; it's useful for converting python3 sources
........
r76161 | benjamin.peterson | 2009-11-08 19:05:37 -0600 (Sun, 08 Nov 2009) | 1 line
simplify condition
........
r76250 | benjamin.peterson | 2009-11-13 16:56:48 -0600 (Fri, 13 Nov 2009) | 1 line
fix handling of a utf-8 bom #7313
........
r76252 | benjamin.peterson | 2009-11-13 16:58:36 -0600 (Fri, 13 Nov 2009) | 1 line
remove pdb turd
........
r76447 | benjamin.peterson | 2009-11-22 18:17:40 -0600 (Sun, 22 Nov 2009) | 1 line
#7375 fix nested transformations in fix_urllib
........
r76506 | benjamin.peterson | 2009-11-24 18:34:31 -0600 (Tue, 24 Nov 2009) | 1 line
use generator expressions in any()
........
Added:
python/trunk/Lib/lib2to3/tests/data/bom.py
- copied unchanged from r76506, /sandbox/trunk/2to3/lib2to3/tests/data/bom.py
Modified:
python/trunk/Lib/lib2to3/ (props changed)
python/trunk/Lib/lib2to3/fixes/fix_imports.py
python/trunk/Lib/lib2to3/fixes/fix_next.py
python/trunk/Lib/lib2to3/fixes/fix_renames.py
python/trunk/Lib/lib2to3/fixes/fix_urllib.py
python/trunk/Lib/lib2to3/main.py
python/trunk/Lib/lib2to3/pgen2/tokenize.py
python/trunk/Lib/lib2to3/refactor.py
python/trunk/Lib/lib2to3/tests/test_fixers.py
python/trunk/Lib/lib2to3/tests/test_refactor.py
Modified: python/trunk/Lib/lib2to3/fixes/fix_imports.py
==============================================================================
--- python/trunk/Lib/lib2to3/fixes/fix_imports.py (original)
+++ python/trunk/Lib/lib2to3/fixes/fix_imports.py Wed Nov 25 19:16:46 2009
@@ -108,7 +108,7 @@
# Module usage could be in the trailer of an attribute lookup, so we
# might have nested matches when "bare_with_attr" is present.
if "bare_with_attr" not in results and \
- any([match(obj) for obj in attr_chain(node, "parent")]):
+ any(match(obj) for obj in attr_chain(node, "parent")):
return False
return results
return False
Modified: python/trunk/Lib/lib2to3/fixes/fix_next.py
==============================================================================
--- python/trunk/Lib/lib2to3/fixes/fix_next.py (original)
+++ python/trunk/Lib/lib2to3/fixes/fix_next.py Wed Nov 25 19:16:46 2009
@@ -99,4 +99,4 @@
def is_subtree(root, node):
if root == node:
return True
- return any([is_subtree(c, node) for c in root.children])
+ return any(is_subtree(c, node) for c in root.children)
Modified: python/trunk/Lib/lib2to3/fixes/fix_renames.py
==============================================================================
--- python/trunk/Lib/lib2to3/fixes/fix_renames.py (original)
+++ python/trunk/Lib/lib2to3/fixes/fix_renames.py Wed Nov 25 19:16:46 2009
@@ -49,7 +49,7 @@
match = super(FixRenames, self).match
results = match(node)
if results:
- if any([match(obj) for obj in attr_chain(node, "parent")]):
+ if any(match(obj) for obj in attr_chain(node, "parent")):
return False
return results
return False
Modified: python/trunk/Lib/lib2to3/fixes/fix_urllib.py
==============================================================================
--- python/trunk/Lib/lib2to3/fixes/fix_urllib.py (original)
+++ python/trunk/Lib/lib2to3/fixes/fix_urllib.py Wed Nov 25 19:16:46 2009
@@ -63,7 +63,8 @@
yield """import_name< 'import'
dotted_as_name< module_as=%r 'as' any > >
""" % old_module
- yield """power< module_dot=%r trailer< '.' member=%s > any* >
+ # bare_with_attr has a special significance for FixImports.match().
+ yield """power< bare_with_attr=%r trailer< '.' member=%s > any* >
""" % (old_module, members)
@@ -150,12 +151,11 @@
def transform_dot(self, node, results):
"""Transform for calls to module members in code."""
- module_dot = results.get('module_dot')
+ module_dot = results.get('bare_with_attr')
member = results.get('member')
- # this may be a list of length one, or just a node
+ new_name = None
if isinstance(member, list):
member = member[0]
- new_name = None
for change in MAPPING[module_dot.value]:
if member.value in change[1]:
new_name = change[0]
@@ -171,7 +171,7 @@
self.transform_import(node, results)
elif results.get('mod_member'):
self.transform_member(node, results)
- elif results.get('module_dot'):
+ elif results.get('bare_with_attr'):
self.transform_dot(node, results)
# Renaming and star imports are not supported for these modules.
elif results.get('module_star'):
Modified: python/trunk/Lib/lib2to3/main.py
==============================================================================
--- python/trunk/Lib/lib2to3/main.py (original)
+++ python/trunk/Lib/lib2to3/main.py Wed Nov 25 19:16:46 2009
@@ -91,8 +91,7 @@
parser.add_option("-l", "--list-fixes", action="store_true",
help="List available transformations (fixes/fix_*.py)")
parser.add_option("-p", "--print-function", action="store_true",
- help="DEPRECATED Modify the grammar so that print() is "
- "a function")
+ help="Modify the grammar so that print() is a function")
parser.add_option("-v", "--verbose", action="store_true",
help="More verbose logging")
parser.add_option("--no-diffs", action="store_true",
@@ -104,12 +103,10 @@
# Parse command line arguments
refactor_stdin = False
+ flags = {}
options, args = parser.parse_args(args)
if not options.write and options.no_diffs:
warn("not writing files and not printing diffs; that's not very useful")
- if options.print_function:
- warn("-p is deprecated; "
- "detection of from __future__ import print_function is automatic")
if not options.write and options.nobackups:
parser.error("Can't use -n without -w")
if options.list_fixes:
@@ -127,6 +124,8 @@
if options.write:
print >> sys.stderr, "Can't write to stdin."
return 2
+ if options.print_function:
+ flags["print_function"] = True
# Set up logging handler
level = logging.DEBUG if options.verbose else logging.INFO
@@ -147,7 +146,7 @@
else:
requested = avail_fixes.union(explicit)
fixer_names = requested.difference(unwanted_fixes)
- rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit),
+ rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
options.nobackups, not options.no_diffs)
# Refactor all files and directories passed as arguments
Modified: python/trunk/Lib/lib2to3/pgen2/tokenize.py
==============================================================================
--- python/trunk/Lib/lib2to3/pgen2/tokenize.py (original)
+++ python/trunk/Lib/lib2to3/pgen2/tokenize.py Wed Nov 25 19:16:46 2009
@@ -281,9 +281,13 @@
# This behaviour mimics the Python interpreter
raise SyntaxError("unknown encoding: " + encoding)
- if bom_found and codec.name != 'utf-8':
- # This behaviour mimics the Python interpreter
- raise SyntaxError('encoding problem: utf-8')
+ if bom_found:
+ if codec.name != 'utf-8':
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError('encoding problem: utf-8')
+ else:
+ # Allow it to be properly encoded and decoded.
+ encoding = 'utf-8-sig'
return encoding
first = read_or_stop()
Modified: python/trunk/Lib/lib2to3/refactor.py
==============================================================================
--- python/trunk/Lib/lib2to3/refactor.py (original)
+++ python/trunk/Lib/lib2to3/refactor.py Wed Nov 25 19:16:46 2009
@@ -18,7 +18,6 @@
import operator
import collections
import StringIO
-import warnings
from itertools import chain
# Local imports
@@ -139,26 +138,23 @@
if have_docstring:
break
have_docstring = True
- elif tp == token.NAME:
- if value == u"from":
+ elif tp == token.NAME and value == u"from":
+ tp, value = advance()
+ if tp != token.NAME and value != u"__future__":
+ break
+ tp, value = advance()
+ if tp != token.NAME and value != u"import":
+ break
+ tp, value = advance()
+ if tp == token.OP and value == u"(":
tp, value = advance()
- if tp != token.NAME and value != u"__future__":
- break
+ while tp == token.NAME:
+ if value == u"print_function":
+ return True
tp, value = advance()
- if tp != token.NAME and value != u"import":
+ if tp != token.OP and value != u",":
break
tp, value = advance()
- if tp == token.OP and value == u"(":
- tp, value = advance()
- while tp == token.NAME:
- if value == u"print_function":
- return True
- tp, value = advance()
- if tp != token.OP and value != u",":
- break
- tp, value = advance()
- else:
- break
else:
break
except StopIteration:
@@ -172,7 +168,7 @@
class RefactoringTool(object):
- _default_options = {}
+ _default_options = {"print_function" : False}
CLASS_PREFIX = "Fix" # The prefix for fixer classes
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -189,15 +185,16 @@
self.explicit = explicit or []
self.options = self._default_options.copy()
if options is not None:
- if "print_function" in options:
- warnings.warn("the 'print_function' option is deprecated",
- DeprecationWarning)
self.options.update(options)
+ if self.options["print_function"]:
+ self.grammar = pygram.python_grammar_no_print_statement
+ else:
+ self.grammar = pygram.python_grammar
self.errors = []
self.logger = logging.getLogger("RefactoringTool")
self.fixer_log = []
self.wrote = False
- self.driver = driver.Driver(pygram.python_grammar,
+ self.driver = driver.Driver(self.grammar,
convert=pytree.convert,
logger=self.logger)
self.pre_order, self.post_order = self.get_fixers()
@@ -353,7 +350,7 @@
name, err.__class__.__name__, err)
return
finally:
- self.driver.grammar = pygram.python_grammar
+ self.driver.grammar = self.grammar
self.log_debug("Refactoring %s", name)
self.refactor_tree(tree, name)
return tree
Modified: python/trunk/Lib/lib2to3/tests/test_fixers.py
==============================================================================
--- python/trunk/Lib/lib2to3/tests/test_fixers.py (original)
+++ python/trunk/Lib/lib2to3/tests/test_fixers.py Wed Nov 25 19:16:46 2009
@@ -1753,6 +1753,8 @@
for old, changes in self.modules.items():
for new, members in changes:
for member in members:
+ new_import = ", ".join([n for (n, mems)
+ in self.modules[old]])
b = """
import %s
foo(%s.%s)
@@ -1760,9 +1762,16 @@
a = """
import %s
foo(%s.%s)
- """ % (", ".join([n for (n, mems)
- in self.modules[old]]),
- new, member)
+ """ % (new_import, new, member)
+ self.check(b, a)
+ b = """
+ import %s
+ %s.%s(%s.%s)
+ """ % (old, old, member, old, member)
+ a = """
+ import %s
+ %s.%s(%s.%s)
+ """ % (new_import, new, member, new, member)
self.check(b, a)
Modified: python/trunk/Lib/lib2to3/tests/test_refactor.py
==============================================================================
--- python/trunk/Lib/lib2to3/tests/test_refactor.py (original)
+++ python/trunk/Lib/lib2to3/tests/test_refactor.py Wed Nov 25 19:16:46 2009
@@ -4,6 +4,7 @@
import sys
import os
+import codecs
import operator
import StringIO
import tempfile
@@ -45,12 +46,10 @@
return refactor.RefactoringTool(fixers, options, explicit)
def test_print_function_option(self):
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always", DeprecationWarning)
- refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True})
- self.assertEqual(len(w), 1)
- msg, = w
- self.assertTrue(msg.category is DeprecationWarning)
+ rt = self.rt({"print_function" : True})
+ self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement)
+ self.assertTrue(rt.driver.grammar is
+ pygram.python_grammar_no_print_statement)
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
@@ -179,10 +178,12 @@
try:
rt.refactor_file(test_file, True)
- self.assertNotEqual(old_contents, read_file())
+ new_contents = read_file()
+ self.assertNotEqual(old_contents, new_contents)
finally:
with open(test_file, "wb") as fp:
fp.write(old_contents)
+ return new_contents
def test_refactor_file(self):
test_file = os.path.join(FIXER_DIR, "parrot_example.py")
@@ -223,6 +224,11 @@
fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
self.check_file_refactoring(fn)
+ def test_bom(self):
+ fn = os.path.join(TEST_DATA_DIR, "bom.py")
+ data = self.check_file_refactoring(fn)
+ self.assertTrue(data.startswith(codecs.BOM_UTF8))
+
def test_crlf_newlines(self):
old_sep = os.linesep
os.linesep = "\r\n"
More information about the Python-checkins
mailing list