[Python-checkins] closes bpo-27494: Fix 2to3 handling of trailing comma after a generator expression (#3771)
Benjamin Peterson
webhook-mailer at python.org
Thu Oct 5 03:10:11 EDT 2017
https://github.com/python/cpython/commit/af810b35b494ef1d255d4bf340b92a9dad446995
commit: af810b35b494ef1d255d4bf340b92a9dad446995
branch: master
author: Jakub Stasiak <jakub at stasiak.at>
committer: Benjamin Peterson <benjamin at python.org>
date: 2017-10-05T00:10:09-07:00
summary:
closes bpo-27494: Fix 2to3 handling of trailing comma after a generator expression (#3771)
files:
A Misc/NEWS.d/next/Library/2017-09-26-01-43-17.bpo-27494.37QnaT.rst
M Lib/lib2to3/Grammar.txt
M Lib/lib2to3/fixer_util.py
M Lib/lib2to3/fixes/fix_dict.py
M Lib/lib2to3/fixes/fix_paren.py
M Lib/lib2to3/fixes/fix_xrange.py
M Lib/lib2to3/tests/test_parser.py
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index 2abd5ee65b5..ded032522ba 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -130,8 +130,8 @@ atom: ('(' [yield_expr|testlist_gexp] ')' |
'{' [dictsetmaker] '}' |
'`' testlist1 '`' |
NAME | NUMBER | STRING+ | '.' '.' '.')
-listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
-testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+listmaker: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
+testlist_gexp: (test|star_expr) ( old_comp_for | (',' (test|star_expr))* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
@@ -161,9 +161,28 @@ argument: ( test [comp_for] |
star_expr )
comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter]
+comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' old_test [comp_iter]
+# As noted above, testlist_safe extends the syntax allowed in list
+# comprehensions and generators. We can't use it indiscriminately in all
+# derivations using a comp_for-like pattern because the testlist_safe derivation
+# contains comma which clashes with trailing comma in arglist.
+#
+# This was an issue because the parser would not follow the correct derivation
+# when parsing syntactically valid Python code. Since testlist_safe was created
+# specifically to handle list comprehensions and generator expressions enclosed
+# with parentheses, it's safe to only use it in those. That avoids the issue; we
+# can parse code like set(x for x in [],).
+#
+# The syntax supported by this set of rules is not a valid Python 3 syntax,
+# hence the prefix "old".
+#
+# See https://bugs.python.org/issue27494
+old_comp_iter: old_comp_for | old_comp_if
+old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
+old_comp_if: 'if' old_test [old_comp_iter]
+
testlist1: test (',' test)*
# not used in grammar, but may appear in "node" passed from Parser to Compiler
diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py
index babe6cb3f66..2f9b1c2e766 100644
--- a/Lib/lib2to3/fixer_util.py
+++ b/Lib/lib2to3/fixer_util.py
@@ -101,8 +101,8 @@ def ListComp(xp, fp, it, test=None):
test.prefix = " "
if_leaf = Leaf(token.NAME, "if")
if_leaf.prefix = " "
- inner_args.append(Node(syms.comp_if, [if_leaf, test]))
- inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)])
+ inner_args.append(Node(syms.old_comp_if, [if_leaf, test]))
+ inner = Node(syms.listmaker, [xp, Node(syms.old_comp_for, inner_args)])
return Node(syms.atom,
[Leaf(token.LBRACE, "["),
inner,
@@ -208,7 +208,7 @@ def attr_chain(obj, attr):
next = getattr(next, attr)
p0 = """for_stmt< 'for' any 'in' node=any ':' any* >
- | comp_for< 'for' any 'in' node=any any* >
+ | old_comp_for< 'for' any 'in' node=any any* >
"""
p1 = """
power<
diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py
index d3655c9f1b2..55be553baac 100644
--- a/Lib/lib2to3/fixes/fix_dict.py
+++ b/Lib/lib2to3/fixes/fix_dict.py
@@ -83,7 +83,7 @@ def transform(self, node, results):
p1 = patcomp.compile_pattern(P1)
P2 = """for_stmt< 'for' any 'in' node=any ':' any* >
- | comp_for< 'for' any 'in' node=any any* >
+ | old_comp_for< 'for' any 'in' node=any any* >
"""
p2 = patcomp.compile_pattern(P2)
diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py
index b205aa7e1e9..de49eef157d 100644
--- a/Lib/lib2to3/fixes/fix_paren.py
+++ b/Lib/lib2to3/fixes/fix_paren.py
@@ -15,7 +15,7 @@ class FixParen(fixer_base.BaseFix):
PATTERN = """
atom< ('[' | '(')
(listmaker< any
- comp_for<
+ old_comp_for<
'for' NAME 'in'
target=testlist_safe< any (',' any)+ [',']
>
@@ -24,7 +24,7 @@ class FixParen(fixer_base.BaseFix):
>
|
testlist_gexp< any
- comp_for<
+ old_comp_for<
'for' NAME 'in'
target=testlist_safe< any (',' any)+ [',']
>
diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py
index 1e491e166a3..f5f06f35435 100644
--- a/Lib/lib2to3/fixes/fix_xrange.py
+++ b/Lib/lib2to3/fixes/fix_xrange.py
@@ -55,7 +55,7 @@ def transform_range(self, node, results):
p1 = patcomp.compile_pattern(P1)
P2 = """for_stmt< 'for' any 'in' node=any ':' any* >
- | comp_for< 'for' any 'in' node=any any* >
+ | old_comp_for< 'for' any 'in' node=any any* >
| comparison< any 'in' node=any any*>
"""
p2 = patcomp.compile_pattern(P2)
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 3f7ab9714e3..2efcb80c2f9 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -459,6 +459,13 @@ def test_multiline_str_literals(self):
self.validate(s)
+class TestGeneratorExpressions(GrammarTest):
+
+ def test_trailing_comma_after_generator_expression_argument_works(self):
+ # BPO issue 27494
+ self.validate("set(x for x in [],)")
+
+
def diff(fn, result):
try:
with open('@', 'w') as f:
diff --git a/Misc/NEWS.d/next/Library/2017-09-26-01-43-17.bpo-27494.37QnaT.rst b/Misc/NEWS.d/next/Library/2017-09-26-01-43-17.bpo-27494.37QnaT.rst
new file mode 100644
index 00000000000..5b5362a41c5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-09-26-01-43-17.bpo-27494.37QnaT.rst
@@ -0,0 +1,2 @@
+Make 2to3 accept a trailing comma in generator expressions. For example, ``set(x
+for x in [],)`` is now allowed.
More information about the Python-checkins
mailing list