[Python-checkins] bpo-36541: Add lib2to3 grammar PEP-570 pos-only arg parsing (GH-23759)

[gpshead]

https://github.com/python/cpython/commit/42c9f0fd0a5e67d4ae0022bfd7370cb9725a5b01
commit: 42c9f0fd0a5e67d4ae0022bfd7370cb9725a5b01
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: gpshead <greg at krypto.org>
date: 2020-12-14T09:10:10-08:00
summary:

bpo-36541: Add lib2to3 grammar PEP-570 pos-only arg parsing (GH-23759)

Add positional only args support to lib2to3 pgen2.

This adds 3.8's PEP-570 support to lib2to3's pgen2.  lib2to3, while
being deprecated is still used by things to parse all versions of Python
code today.  We need it to support parsing modern 3.8 and 3.9 constructs.

Also add tests for complex *expr and **expr's.

files:
A Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst
M Lib/lib2to3/Grammar.txt
M Lib/lib2to3/tests/test_parser.py

diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index e007dc188af50..fa7b15061d941 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -18,15 +18,55 @@ decorated: decorators (classdef | funcdef | async_funcdef)
 async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
-typedargslist: ((tfpdef ['=' test] ',')*
-                ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [','])
-                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+
+# The following definition for typedarglist is equivalent to this set of rules:
+#
+#     arguments = argument (',' argument)*
+#     argument = tfpdef ['=' test]
+#     kwargs = '**' tname [',']
+#     args = '*' [tname]
+#     kwonly_kwargs = (',' argument)* [',' [kwargs]]
+#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
+#     poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
+#     typedargslist_no_posonly  = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
+#     typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)"
+#
+# It needs to be fully expanded to allow our LL(1) parser to work on it.
+
+typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [
+                     ',' [((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])*
+                            [',' ['**' tname [',']]] | '**' tname [','])
+                     | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])]
+                ] | ((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])*
+                     [',' ['**' tname [',']]] | '**' tname [','])
+                     | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+
 tname: NAME [':' test]
 tfpdef: tname | '(' tfplist ')'
 tfplist: tfpdef (',' tfpdef)* [',']
-varargslist: ((vfpdef ['=' test] ',')*
-              ('*' [vname] (',' vname ['=' test])*  [',' ['**' vname [',']]] | '**' vname [','])
-              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+
+# The following definition for varargslist is equivalent to this set of rules:
+#
+#     arguments = argument (',' argument )*
+#     argument = vfpdef ['=' test]
+#     kwargs = '**' vname [',']
+#     args = '*' [vname]
+#     kwonly_kwargs = (',' argument )* [',' [kwargs]]
+#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
+#     poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
+#     vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
+#     varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly)
+#
+# It needs to be fully expanded to allow our LL(1) parser to work on it.
+
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [
+                     ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])*
+                            [',' ['**' vname [',']]] | '**' vname [','])
+                            | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+                     ]] | ((vfpdef ['=' test] ',')*
+                     ('*' [vname] (',' vname ['=' test])*  [',' ['**' vname [',']]]| '**' vname [','])
+                     | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+
 vname: NAME
 vfpdef: vname | '(' vfplist ')'
 vfplist: vfpdef (',' vfpdef)* [',']
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index ba2bb787332ed..d5db66b9b1e7b 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -272,6 +272,12 @@ def test_dict_display_1(self):
     def test_dict_display_2(self):
         self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
 
+    def test_complex_star_expression(self):
+        self.validate("func(* [] or [1])")
+
+    def test_complex_double_star_expression(self):
+        self.validate("func(**{1: 3} if False else {x: x for x in range(3)})")
+
     def test_argument_unpacking_1(self):
         self.validate("""f(a, *b, *c, d)""")
 
@@ -630,6 +636,7 @@ def test_multiline_str_literals(self):
 
 
 class TestNamedAssignments(GrammarTest):
+    """Also known as the walrus operator."""
 
     def test_named_assignment_if(self):
         driver.parse_string("if f := x(): pass\n")
@@ -644,6 +651,30 @@ def test_named_assignment_listcomp(self):
         driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n")
 
 
+class TestPositionalOnlyArgs(GrammarTest):
+
+    def test_one_pos_only_arg(self):
+        driver.parse_string("def one_pos_only_arg(a, /): pass\n")
+
+    def test_all_markers(self):
+        driver.parse_string(
+                "def all_markers(a, b=2, /, c, d=4, *, e=5, f): pass\n")
+
+    def test_all_with_args_and_kwargs(self):
+        driver.parse_string(
+                """def all_markers_with_args_and_kwargs(
+                           aa, b, /, _cc, d, *args, e, f_f, **kwargs,
+                   ):
+                       pass\n""")
+
+    def test_lambda_soup(self):
+        driver.parse_string(
+                "lambda a, b, /, c, d, *args, e, f, **kw: kw\n")
+
+    def test_only_positional_or_keyword(self):
+        driver.parse_string("def func(a,b,/,*,g,e=3): pass\n")
+
+
 class TestPickleableException(unittest.TestCase):
     def test_ParseError(self):
         err = ParseError('msg', 2, None, (1, 'context'))
diff --git a/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst b/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst
new file mode 100644
index 0000000000000..5678d8c595ca0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst
@@ -0,0 +1,2 @@
+Fixed lib2to3.pgen2 to be able to parse PEP-570 positional only argument
+syntax.