[Python-checkins] bpo-33645: Fix an "unknown parsing error" in the parser. (GH-7119)

[Serhiy Storchaka]

https://github.com/python/cpython/commit/d5e7556e522f4662ad34b35924b6c76895df340e
commit: d5e7556e522f4662ad34b35924b6c76895df340e
branch: 2.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2018-05-31T07:35:39+03:00
summary:

bpo-33645: Fix an "unknown parsing error" in the parser. (GH-7119)

It is reproduced when parse the "<>" operator and run
Python with both options -3 and -We.

files:
A Misc/NEWS.d/next/Core and Builtins/2018-05-25-18-20-04.bpo-33645.GYGIPH.rst
M Lib/test/test_grammar.py
M Parser/tokenizer.c

diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 23b6ce8d3c5c..228586ece02a 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -5,6 +5,7 @@
                               check_py3k_warnings
 import unittest
 import sys
+import warnings
 # testing import *
 from sys import *
 
@@ -628,7 +629,6 @@ def check(code, warntext):
             with check_py3k_warnings((warntext, DeprecationWarning)):
                 compile(code, '<test string>', 'exec')
             if sys.py3kwarning:
-                import warnings
                 with warnings.catch_warnings():
                     warnings.filterwarnings('error', category=DeprecationWarning)
                     with self.assertRaises(SyntaxError) as cm:
@@ -883,6 +883,13 @@ def test_comparison(self):
         with check_py3k_warnings(('<> not supported in 3.x; use !=',
                                   DeprecationWarning)):
             if eval('1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1'): pass
+        if sys.py3kwarning:
+            with warnings.catch_warnings():
+                warnings.filterwarnings('error', category=DeprecationWarning)
+                with self.assertRaises(DeprecationWarning) as cm:
+                    compile('1 <> 1', '<test string>', 'eval')
+                self.assertIn('<> not supported in 3.x; use !=',
+                              str(cm.exception))
 
     def test_binary_mask_ops(self):
         x = 1 & 1
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-25-18-20-04.bpo-33645.GYGIPH.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-25-18-20-04.bpo-33645.GYGIPH.rst
new file mode 100644
index 000000000000..5cc47f1532d3
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-25-18-20-04.bpo-33645.GYGIPH.rst	
@@ -0,0 +1,2 @@
+Fixed an "unknown parsing error" on parsing the "<>" operator when run
+Python with both options ``-3`` and ``-We``.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 4fdbad939341..61bfb4e1b7af 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1636,6 +1636,8 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
                                    "<> not supported in 3.x; use !=",
                                    tok->filename, tok->lineno,
                                    NULL, NULL)) {
+                tok->done = E_ERROR;
+                tok->cur = tok->inp;
                 return ERRORTOKEN;
             }
         }