[Python-checkins] make lib2to3 parse async generators everywhere (GH-6588) (GH-27703)

[ambv]

https://github.com/python/cpython/commit/0e63776c4f3d0b972344368f62b91eb96be69fe3
commit: 0e63776c4f3d0b972344368f62b91eb96be69fe3
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: ambv <lukasz at langa.pl>
date: 2021-08-10T11:56:50+02:00
summary:

make lib2to3 parse async generators everywhere (GH-6588) (GH-27703)

(cherry picked from commit 149addd4960d634ce672ab5fc17e0e785a0cdcd0)

Co-authored-by: Zsolt Dollenstein <zsol.zsol at gmail.com>

files:
A Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst
M Lib/lib2to3/pgen2/tokenize.py
M Lib/lib2to3/tests/test_parser.py

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 0e2685d40433d7..099dfa7798afd4 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -512,13 +512,14 @@ def generate_tokens(readline):
                         stashed = tok
                         continue
 
-                    if token == 'def':
+                    if token in ('def', 'for'):
                         if (stashed
                                 and stashed[0] == NAME
                                 and stashed[1] == 'async'):
 
-                            async_def = True
-                            async_def_indent = indents[-1]
+                            if token == 'def':
+                                async_def = True
+                                async_def_indent = indents[-1]
 
                             yield (ASYNC, stashed[1],
                                    stashed[2], stashed[3],
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index d5db66b9b1e7b9..034b5039cf00c4 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -196,20 +196,27 @@ def test_async_var(self):
         self.validate("""await = 1""")
         self.validate("""def async(): pass""")
 
-    def test_async_with(self):
+    def test_async_for(self):
         self.validate("""async def foo():
                              async for a in b: pass""")
 
-        self.invalid_syntax("""def foo():
-                                   async for a in b: pass""")
-
-    def test_async_for(self):
+    def test_async_with(self):
         self.validate("""async def foo():
                              async with a: pass""")
 
         self.invalid_syntax("""def foo():
                                    async with a: pass""")
 
+    def test_async_generator(self):
+        self.validate(
+            """async def foo():
+                   return (i * 2 async for i in arange(42))"""
+        )
+        self.validate(
+            """def foo():
+                   return (i * 2 async for i in arange(42))"""
+        )
+
 
 class TestRaiseChanges(GrammarTest):
     def test_2x_style_1(self):
diff --git a/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst b/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst
new file mode 100644
index 00000000000000..be68b3ea7c4a2a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst
@@ -0,0 +1 @@
+lib2to3 now recognizes async generators everywhere.