[Python-checkins] Fix 'gather' rules in the python parser generator (GH-22021)

Pablo Galindo webhook-mailer at python.org
Thu Sep 3 10:29:59 EDT 2020


https://github.com/python/cpython/commit/e55a0e971b4bf9e6473bf0ca6bebdff76c075ef6
commit: e55a0e971b4bf9e6473bf0ca6bebdff76c075ef6
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020-09-03T15:29:55+01:00
summary:

Fix 'gather' rules in the python parser generator (GH-22021)

Currently, empty sequences in gather rules make the conditional for
gather rules fail as empty sequences evaluate as "False". We need to
explicitly check for "None" (the failure condition) to avoid false
negatives.

files:
M Lib/test/test_peg_generator/test_pegen.py
M Tools/peg_generator/pegen/python_generator.py

diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 5b4e964d698ad..bcfee3f2c5f8c 100644
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -74,7 +74,7 @@ def test_typed_rules(self) -> None:
             "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))"
         )
 
-    def test_repeat_with_separator_rules(self) -> None:
+    def test_gather(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
@@ -85,6 +85,20 @@ def test_repeat_with_separator_rules(self) -> None:
             "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
         ))
         self.assertEqual(str(rules["thing"]), "thing: NUMBER")
+        parser_class = make_parser(grammar)
+        node = parse_string("42\n", parser_class)
+        assert node == [
+            [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
+            TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
+        ]
+        node = parse_string("1, 2\n", parser_class)
+        assert node == [
+            [
+                [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")],
+                [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")],
+            ],
+            TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"),
+        ]
 
     def test_expr_grammar(self) -> None:
         grammar = """
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 45a75975dbf5e..b786de7fee5b4 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -217,6 +217,9 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None:
                     else:
                         self.print("and")
                     self.visit(item)
+                    if is_gather:
+                        self.print("is not None")
+
             self.print("):")
             with self.indent():
                 action = node.action



More information about the Python-checkins mailing list