[Python-checkins] r73981 - in sandbox/trunk/2to3/lib2to3: fixes/fix_print.py main.py pgen2/grammar.py pygram.py refactor.py tests/test_fixers.py tests/test_refactor.py
benjamin.peterson
python-checkins at python.org
Sun Jul 12 19:06:40 CEST 2009
Author: benjamin.peterson
Date: Sun Jul 12 19:06:39 2009
New Revision: 73981
Log:
detect when "from __future__ import print_function" is given
Deprecate the 'print_function' option and the -p flag
Modified:
sandbox/trunk/2to3/lib2to3/fixes/fix_print.py
sandbox/trunk/2to3/lib2to3/main.py
sandbox/trunk/2to3/lib2to3/pgen2/grammar.py
sandbox/trunk/2to3/lib2to3/pygram.py
sandbox/trunk/2to3/lib2to3/refactor.py
sandbox/trunk/2to3/lib2to3/tests/test_fixers.py
sandbox/trunk/2to3/lib2to3/tests/test_refactor.py
Modified: sandbox/trunk/2to3/lib2to3/fixes/fix_print.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/fixes/fix_print.py (original)
+++ sandbox/trunk/2to3/lib2to3/fixes/fix_print.py Sun Jul 12 19:06:39 2009
@@ -26,20 +26,15 @@
)
-class FixPrint(fixer_base.ConditionalFix):
+class FixPrint(fixer_base.BaseFix):
PATTERN = """
simple_stmt< any* bare='print' any* > | print_stmt
"""
- skip_on = '__future__.print_function'
-
def transform(self, node, results):
assert results
- if self.should_skip(node):
- return
-
bare_print = results.get("bare")
if bare_print:
Modified: sandbox/trunk/2to3/lib2to3/main.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/main.py (original)
+++ sandbox/trunk/2to3/lib2to3/main.py Sun Jul 12 19:06:39 2009
@@ -91,7 +91,8 @@
parser.add_option("-l", "--list-fixes", action="store_true",
help="List available transformations (fixes/fix_*.py)")
parser.add_option("-p", "--print-function", action="store_true",
- help="Modify the grammar so that print() is a function")
+ help="DEPRECATED Modify the grammar so that print() is "
+ "a function")
parser.add_option("-v", "--verbose", action="store_true",
help="More verbose logging")
parser.add_option("--no-diffs", action="store_true",
@@ -106,6 +107,9 @@
options, args = parser.parse_args(args)
if not options.write and options.no_diffs:
warn("not writing files and not printing diffs; that's not very useful")
+ if options.print_function:
+ warn("-p is deprecated; "
+ "detection of from __future__ import print_function is automatic")
if not options.write and options.nobackups:
parser.error("Can't use -n without -w")
if options.list_fixes:
@@ -129,7 +133,6 @@
logging.basicConfig(format='%(name)s: %(message)s', level=level)
# Initialize the refactoring tool
- rt_opts = {"print_function" : options.print_function}
avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)
explicit = set()
@@ -144,7 +147,7 @@
else:
requested = avail_fixes.union(explicit)
fixer_names = requested.difference(unwanted_fixes)
- rt = StdoutRefactoringTool(sorted(fixer_names), rt_opts, sorted(explicit),
+ rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit),
options.nobackups, not options.no_diffs)
# Refactor all files and directories passed as arguments
Modified: sandbox/trunk/2to3/lib2to3/pgen2/grammar.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/pgen2/grammar.py (original)
+++ sandbox/trunk/2to3/lib2to3/pgen2/grammar.py Sun Jul 12 19:06:39 2009
@@ -97,6 +97,19 @@
f.close()
self.__dict__.update(d)
+ def copy(self):
+ """
+ Copy the grammar.
+ """
+ new = self.__class__()
+ for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
+ "tokens", "symbol2label"):
+ setattr(new, dict_attr, getattr(self, dict_attr).copy())
+ new.labels = self.labels[:]
+ new.states = self.states[:]
+ new.start = self.start
+ return new
+
def report(self):
"""Dump the grammar tables to standard output, for debugging."""
from pprint import pprint
Modified: sandbox/trunk/2to3/lib2to3/pygram.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/pygram.py (original)
+++ sandbox/trunk/2to3/lib2to3/pygram.py Sun Jul 12 19:06:39 2009
@@ -28,4 +28,8 @@
python_grammar = driver.load_grammar(_GRAMMAR_FILE)
+
python_symbols = Symbols(python_grammar)
+
+python_grammar_no_print_statement = python_grammar.copy()
+del python_grammar_no_print_statement.keywords["print"]
Modified: sandbox/trunk/2to3/lib2to3/refactor.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/refactor.py (original)
+++ sandbox/trunk/2to3/lib2to3/refactor.py Sun Jul 12 19:06:39 2009
@@ -17,10 +17,12 @@
import logging
import operator
import collections
+import StringIO
+import warnings
from itertools import chain
# Local imports
-from .pgen2 import driver, tokenize
+from .pgen2 import driver, tokenize, token
from . import pytree, pygram
@@ -121,13 +123,56 @@
_to_system_newlines = _identity
+def _detect_future_print(source):
+ have_docstring = False
+ gen = tokenize.generate_tokens(StringIO.StringIO(source).readline)
+ def advance():
+ tok = next(gen)
+ return tok[0], tok[1]
+ ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+ try:
+ while True:
+ tp, value = advance()
+ if tp in ignore:
+ continue
+ elif tp == token.STRING:
+ if have_docstring:
+ break
+ have_docstring = True
+ elif tp == token.NAME:
+ if value == u"from":
+ tp, value = advance()
+ if tp != token.NAME and value != u"__future__":
+ break
+ tp, value = advance()
+ if tp != token.NAME and value != u"import":
+ break
+ tp, value = advance()
+ if tp == token.OP and value == u"(":
+ tp, value = advance()
+ while tp == token.NAME:
+ if value == u"print_function":
+ return True
+ tp, value = advance()
+ if tp != token.OP and value != u",":
+ break
+ tp, value = advance()
+ else:
+ break
+ else:
+ break
+ except StopIteration:
+ pass
+ return False
+
+
class FixerError(Exception):
"""A fixer could not be loaded."""
class RefactoringTool(object):
- _default_options = {"print_function": False}
+ _default_options = {}
CLASS_PREFIX = "Fix" # The prefix for fixer classes
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -144,13 +189,14 @@
self.explicit = explicit or []
self.options = self._default_options.copy()
if options is not None:
+ if "print_function" in options:
+ warnings.warn("the 'print_function' option is deprecated",
+ DeprecationWarning)
self.options.update(options)
self.errors = []
self.logger = logging.getLogger("RefactoringTool")
self.fixer_log = []
self.wrote = False
- if self.options["print_function"]:
- del pygram.python_grammar.keywords["print"]
self.driver = driver.Driver(pygram.python_grammar,
convert=pytree.convert,
logger=self.logger)
@@ -298,12 +344,16 @@
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
"""
+ if _detect_future_print(data):
+ self.driver.grammar = pygram.python_grammar_no_print_statement
try:
tree = self.driver.parse_string(data)
except Exception, err:
self.log_error("Can't parse %s: %s: %s",
name, err.__class__.__name__, err)
return
+ finally:
+ self.driver.grammar = pygram.python_grammar
self.log_debug("Refactoring %s", name)
self.refactor_tree(tree, name)
return tree
Modified: sandbox/trunk/2to3/lib2to3/tests/test_fixers.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/tests/test_fixers.py (original)
+++ sandbox/trunk/2to3/lib2to3/tests/test_fixers.py Sun Jul 12 19:06:39 2009
@@ -379,18 +379,15 @@
self.unchanged(s)
def test_idempotency_print_as_function(self):
- print_stmt = pygram.python_grammar.keywords.pop("print")
- try:
- s = """print(1, 1+1, 1+1+1)"""
- self.unchanged(s)
+ self.refactor.driver.grammar = pygram.python_grammar_no_print_statement
+ s = """print(1, 1+1, 1+1+1)"""
+ self.unchanged(s)
- s = """print()"""
- self.unchanged(s)
+ s = """print()"""
+ self.unchanged(s)
- s = """print('')"""
- self.unchanged(s)
- finally:
- pygram.python_grammar.keywords["print"] = print_stmt
+ s = """print('')"""
+ self.unchanged(s)
def test_1(self):
b = """print 1, 1+1, 1+1+1"""
@@ -462,29 +459,15 @@
a = """print(file=sys.stderr)"""
self.check(b, a)
- # With from __future__ import print_function
def test_with_future_print_function(self):
- # XXX: These tests won't actually do anything until the parser
- # is fixed so it won't crash when it sees print(x=y).
- # When #2412 is fixed, the try/except block can be taken
- # out and the tests can be run like normal.
- try:
- s = "from __future__ import print_function\n"\
- "print('Hai!', end=' ')"
- self.unchanged(s)
+ s = "from __future__ import print_function\n" \
+ "print('Hai!', end=' ')"
+ self.unchanged(s)
- b = "print 'Hello, world!'"
- a = "print('Hello, world!')"
- self.check(b, a)
+ b = "print 'Hello, world!'"
+ a = "print('Hello, world!')"
+ self.check(b, a)
- s = "from __future__ import *\n"\
- "print('Hai!', end=' ')"
- self.unchanged(s)
- except:
- return
- else:
- self.assertFalse(True, "#2421 has been fixed -- printing tests "\
- "need to be updated!")
class Test_exec(FixerTestCase):
fixer = "exec"
Modified: sandbox/trunk/2to3/lib2to3/tests/test_refactor.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/tests/test_refactor.py (original)
+++ sandbox/trunk/2to3/lib2to3/tests/test_refactor.py Sun Jul 12 19:06:39 2009
@@ -9,6 +9,7 @@
import tempfile
import shutil
import unittest
+import warnings
from lib2to3 import refactor, pygram, fixer_base
from lib2to3.pgen2 import token
@@ -44,14 +45,11 @@
return refactor.RefactoringTool(fixers, options, explicit)
def test_print_function_option(self):
- gram = pygram.python_grammar
- save = gram.keywords["print"]
- try:
- rt = self.rt({"print_function" : True})
- self.assertRaises(KeyError, operator.itemgetter("print"),
- gram.keywords)
- finally:
- gram.keywords["print"] = save
+ with warnings.catch_warnings(record=True) as w:
+ refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True})
+ self.assertEqual(len(w), 1)
+ msg, = w
+ self.assertTrue(msg.category is DeprecationWarning)
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
@@ -63,6 +61,43 @@
self.assertEqual(full_names,
["myfixes.fix_" + name for name in contents])
+ def test_detect_future_print(self):
+ run = refactor._detect_future_print
+ self.assertFalse(run(""))
+ self.assertTrue(run("from __future__ import print_function"))
+ self.assertFalse(run("from __future__ import generators"))
+ self.assertFalse(run("from __future__ import generators, feature"))
+ input = "from __future__ import generators, print_function"
+ self.assertTrue(run(input))
+ input ="from __future__ import print_function, generators"
+ self.assertTrue(run(input))
+ input = "from __future__ import (print_function,)"
+ self.assertTrue(run(input))
+ input = "from __future__ import (generators, print_function)"
+ self.assertTrue(run(input))
+ input = "from __future__ import (generators, nested_scopes)"
+ self.assertFalse(run(input))
+ input = """from __future__ import generators
+from __future__ import print_function"""
+ self.assertTrue(run(input))
+ self.assertFalse(run("from"))
+ self.assertFalse(run("from 4"))
+ self.assertFalse(run("from x"))
+ self.assertFalse(run("from x 5"))
+ self.assertFalse(run("from x im"))
+ self.assertFalse(run("from x import"))
+ self.assertFalse(run("from x import 4"))
+ input = "'docstring'\nfrom __future__ import print_function"
+ self.assertTrue(run(input))
+ input = "'docstring'\n'somng'\nfrom __future__ import print_function"
+ self.assertFalse(run(input))
+ input = "# comment\nfrom __future__ import print_function"
+ self.assertTrue(run(input))
+ input = "# comment\n'doc'\nfrom __future__ import print_function"
+ self.assertTrue(run(input))
+ input = "class x: pass\nfrom __future__ import print_function"
+ self.assertFalse(run(input))
+
def test_get_headnode_dict(self):
class NoneFix(fixer_base.BaseFix):
pass
More information about the Python-checkins
mailing list