[Python-checkins] bpo-46054: Fix parsing error when parsing non-utf8 characters in source files (GH-30068) (GH-30069)

[pablogsal]

https://github.com/python/cpython/commit/94483f1e3cec182fabe19268e579f63045bc984a
commit: 94483f1e3cec182fabe19268e579f63045bc984a
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-12-12T16:52:49Z
summary:

bpo-46054: Fix parsing error when parsing non-utf8 characters in source files (GH-30068) (GH-30069)

(cherry picked from commit 4325a766f5f603ef6dfb8c4d5798e5e73cb5efd5)

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2021-12-12-05-30-21.bpo-46054.2P-foG.rst
M Lib/test/test_exceptions.py
M Parser/tokenizer.c

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 9acb16c518210..cc0640dda0980 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -2368,6 +2368,18 @@ def test_encodings(self):
         finally:
             unlink(TESTFN)
 
+    def test_non_utf8(self):
+        # Check non utf-8 characters
+        try:
+            with open(TESTFN, 'bw') as testfile:
+                testfile.write(b"\x89")
+            rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN)
+            err = err.decode('utf-8').splitlines()
+
+            self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1])
+        finally:
+            unlink(TESTFN)
+
     def test_attributes_new_constructor(self):
         args = ("bad.py", 1, 2, "abcdefg", 1, 100)
         the_exception = SyntaxError("bad bad", args)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-12-05-30-21.bpo-46054.2P-foG.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-12-05-30-21.bpo-46054.2P-foG.rst
new file mode 100644
index 0000000000000..6ca91f03445e2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-12-05-30-21.bpo-46054.2P-foG.rst	
@@ -0,0 +1,2 @@
+Fix parser error when parsing non-utf8 characters in source files. Patch by
+Pablo Galindo.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 672fdb92ec86f..8e9c69d0785af 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -818,10 +818,10 @@ tok_readline_raw(struct tok_state *tok)
             tok_concatenate_interactive_new_line(tok, line) == -1) {
             return 0;
         }
-        if (*tok->inp == '\0') {
+        tok->inp = strchr(tok->inp, '\0');
+        if (tok->inp == tok->buf) {
             return 0;
         }
-        tok->inp = strchr(tok->inp, '\0');
     } while (tok->inp[-1] != '\n');
     return 1;
 }
@@ -983,12 +983,9 @@ tok_underflow_file(struct tok_state *tok) {
     }
     /* The default encoding is UTF-8, so make sure we don't have any
        non-UTF-8 sequences in it. */
-    if (!tok->encoding
-        && (tok->decoding_state != STATE_NORMAL || tok->lineno >= 2)) {
-        if (!ensure_utf8(tok->cur, tok)) {
-            error_ret(tok);
-            return 0;
-        }
+    if (!tok->encoding && !ensure_utf8(tok->cur, tok)) {
+        error_ret(tok);
+        return 0;
     }
     assert(tok->done == E_OK);
     return tok->done == E_OK;