[Python-checkins] bpo-40334: Fix test_peg_parser to actually use the old parser (GH-19778)

[Lysandros Nikolaou]

https://github.com/python/cpython/commit/69e802ed812e38cb68a4ab74af64b4f719b6cc78
commit: 69e802ed812e38cb68a4ab74af64b4f719b6cc78
branch: master
author: Lysandros Nikolaou <lisandrosnik at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020-04-29T23:53:30+01:00
summary:

bpo-40334: Fix test_peg_parser to actually use the old parser (GH-19778)

Now that the default parser is the new PEG parser, ast.parse uses it, which means that we don't actually test something in test_peg_parser. This commit introduces a new keyword argument (`oldparser`) for `_peg_parser.parse_string` for specifying that a string needs to be parsed with the old parser. This keyword argument is used in the tests to actually compare the ASTs the new parser generates with those generated by the old parser.

files:
M Lib/test/test_peg_parser.py
M Modules/_peg_parser.c

diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py
index ea4afa6e17973..43d30bae84984 100644
--- a/Lib/test/test_peg_parser.py
+++ b/Lib/test/test_peg_parser.py
@@ -699,7 +699,7 @@ def test_correct_ast_generation_on_source_files(self) -> None:
         self.maxDiff = None
         for source in TEST_SOURCES:
             actual_ast = peg_parser.parse_string(source)
-            expected_ast = ast.parse(source)
+            expected_ast = peg_parser.parse_string(source, oldparser=True)
             self.assertEqual(
                 ast.dump(actual_ast, include_attributes=True),
                 ast.dump(expected_ast, include_attributes=True),
@@ -721,12 +721,11 @@ def test_incorrect_ast_generation_with_specialized_errors(self) -> None:
                 f"Actual error message does not match expexted for {source}"
             )
 
-    @support.skip_if_new_parser("This tests nothing for now, since compile uses pegen as well")
     @unittest.expectedFailure
     def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
             actual_ast = peg_parser.parse_string(source)
-            expected_ast = ast.parse(source)
+            expected_ast = peg_parser.parse_string(source, oldparser=True)
             self.assertEqual(
                 ast.dump(actual_ast, include_attributes=True),
                 ast.dump(expected_ast, include_attributes=True),
@@ -736,7 +735,7 @@ def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
     def test_correct_ast_generation_without_pos_info(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
             actual_ast = peg_parser.parse_string(source)
-            expected_ast = ast.parse(source)
+            expected_ast = peg_parser.parse_string(source, oldparser=True)
             self.assertEqual(
                 ast.dump(actual_ast),
                 ast.dump(expected_ast),
@@ -752,7 +751,7 @@ def test_fstring_parse_error_tracebacks(self) -> None:
     def test_correct_ast_generatrion_eval(self) -> None:
         for source in EXPRESSIONS_TEST_SOURCES:
             actual_ast = peg_parser.parse_string(source, mode='eval')
-            expected_ast = ast.parse(source, mode='eval')
+            expected_ast = peg_parser.parse_string(source, mode='eval', oldparser=True)
             self.assertEqual(
                 ast.dump(actual_ast, include_attributes=True),
                 ast.dump(expected_ast, include_attributes=True),
diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c
index e1ec36e07bd57..59b80f9e06e9e 100644
--- a/Modules/_peg_parser.c
+++ b/Modules/_peg_parser.c
@@ -45,11 +45,13 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds)
 PyObject *
 _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    static char *keywords[] = {"string", "mode", NULL};
+    static char *keywords[] = {"string", "mode", "oldparser", NULL};
     char *the_string;
     char *mode_str = "exec";
+    int oldparser = 0;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|s", keywords, &the_string, &mode_str)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sp", keywords,
+            &the_string, &mode_str, &oldparser)) {
         return NULL;
     }
 
@@ -77,7 +79,13 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds)
     PyCompilerFlags flags = _PyCompilerFlags_INIT;
     flags.cf_flags = PyCF_IGNORE_COOKIE;
 
-    mod_ty res = PyPegen_ASTFromString(the_string, mode, &flags, arena);
+    mod_ty res;
+    if (oldparser) {
+        res = PyParser_ASTFromString(the_string, "<string>", mode, &flags, arena);
+    }
+    else {
+        res = PyPegen_ASTFromString(the_string, mode, &flags, arena);
+    }
     if (res == NULL) {
         goto error;
     }