https://github.com/python/cpython/commit/a23a2c555c4187f349276fe2f2ceffa953d... commit: a23a2c555c4187f349276fe2f2ceffa953d0afe9 branch: 3.6 author: Benjamin Peterson <benjamin@python.org> committer: GitHub <noreply@github.com> date: 2018-01-30T10:23:17-08:00 summary: [3.6] closes bpo-30117: fix lib2to3 ParserIdempotency test (GH-1242) (GH-5443) Fix two (in my opinion) spurious failure conditions in the lib2to3.tests.test_parser.TestParserIdempotency test_parser test. Use the same encoding found in the initial file to write a temp file for a diff. This retains the BOM if the encoding was initially utf-8-sig. If the file cannot be parsed using the normal grammar, try again with no print statement which should succeed for valid files using future print_function For case (1), the driver was correctly handling a BOM in a utf-8 file, but then the test was not writing a comparison file using 'utf-8-sig' to diff against, so the BOM got removed. I don't think that is the fault of the parser, and lib2to3 will retain the BOM. For case (2), lib2to3 pre-detects the use of from __future__ import print_function or allows the user to force this interpretation with a -p flag, and then selects a different grammar with the print statement removed. That makes the test cases unfair to this test as the driver itself doesn't know which grammar to use. As a minimal fix, the test will try using a grammar with the print statement, and if that fails fall back on a grammar without it. A more thorough handling of the idempotency test would to be to parse all files using both grammars and ignore if one of the two failed but otherwise check both. I didn't think this was necessary but can change.. (cherry picked from commit 14e976e00e65bf343ba0fca016c3c9132a843daf) files: M Lib/lib2to3/tests/support.py M Lib/lib2to3/tests/test_parser.py diff --git a/Lib/lib2to3/tests/support.py b/Lib/lib2to3/tests/support.py index ae7cfe8ee274..fe084e8903fc 100644 --- a/Lib/lib2to3/tests/support.py +++ b/Lib/lib2to3/tests/support.py @@ -15,7 +15,13 @@ proj_dir = os.path.normpath(os.path.join(test_dir, "..")) grammar_path = os.path.join(test_dir, "..", "Grammar.txt") grammar = pgen2_driver.load_grammar(grammar_path) +grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path) +del grammar_no_print_statement.keywords["print"] driver = pgen2_driver.Driver(grammar, convert=pytree.convert) +driver_no_print_statement = pgen2_driver.Driver( + grammar_no_print_statement, + convert=pytree.convert +) def parse_string(string): return driver.parse_string(reformat(string), debug=True) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 0a139507b8de..9d444c204aa6 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -8,7 +8,7 @@ # Testing imports from . import support -from .support import driver +from .support import driver, driver_no_print_statement from test.support import verbose # Python imports @@ -413,8 +413,6 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" - # Issue 13125 - @unittest.expectedFailure def test_all_project_files(self): for filepath in support.all_project_files(): with open(filepath, "rb") as fp: @@ -425,12 +423,13 @@ def test_all_project_files(self): source = fp.read() try: tree = driver.parse_string(source) - except ParseError as err: - if verbose > 0: - warnings.warn('ParseError on file %s (%s)' % (filepath, err)) - continue + except ParseError: + try: + tree = driver_no_print_statement.parse_string(source) + except ParseError as err: + self.fail('ParseError on file %s (%s)' % (filepath, err)) new = str(tree) - x = diff(filepath, new) + x = diff(filepath, new, encoding=encoding) if x: self.fail("Idempotency failed: %s" % filepath) @@ -474,9 +473,9 @@ def test_multiline_str_literals(self): self.validate(s) -def diff(fn, result): +def diff(fn, result, encoding='utf-8'): try: - with open('@', 'w') as f: + with open('@', 'w', encoding=encoding, newline='\n') as f: f.write(str(result)) fn = fn.replace('"', '\\"') return subprocess.call(['diff', '-u', fn, '@'], stdout=(subprocess.DEVNULL if verbose < 1 else None))