[Python-checkins] cpython (merge 3.4 -> default): merge 3.4 (#24022)
benjamin.peterson
python-checkins at python.org
Tue Apr 21 18:07:11 CEST 2015
https://hg.python.org/cpython/rev/03b2259c6cd3
changeset: 95751:03b2259c6cd3
parent: 95749:475c6a4dfab3
parent: 95750:414e08c478f4
user: Benjamin Peterson <benjamin at python.org>
date: Tue Apr 21 12:07:06 2015 -0400
summary:
merge 3.4 (#24022)
files:
Lib/test/test_compile.py | 14 +++++++++++++-
Misc/NEWS | 2 ++
Parser/tokenizer.c | 7 +++----
3 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -1,9 +1,11 @@
import math
+import os
import unittest
import sys
import _ast
+import tempfile
import types
-from test import support
+from test import support, script_helper
class TestSpecifics(unittest.TestCase):
@@ -492,6 +494,16 @@
self.assertInvalidSingle('f()\nxy # blah\nblah()')
self.assertInvalidSingle('x = 5 # comment\nx = 6\n')
+ def test_particularly_evil_undecodable(self):
+ # Issue 24022
+ src = b'0000\x00\n00000000000\n\x00\n\x9e\n'
+ with tempfile.TemporaryDirectory() as tmpd:
+ fn = os.path.join(tmpd, "bad.py")
+ with open(fn, "wb") as fp:
+ fp.write(src)
+ res = script_helper.run_python_until_end(fn)[0]
+ self.assertIn(b"Non-UTF-8", res.err)
+
@support.cpython_only
def test_compiler_recursion_limit(self):
# Expected limit is sys.getrecursionlimit() * the scaling factor
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
Core and Builtins
-----------------
+- Issue #24022: Fix tokenizer crash when processing undecodable source code.
+
Library
-------
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1307,6 +1307,8 @@
{
PyObject *s;
int result;
+ if (tok->decoding_erred)
+ return 0;
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
if (s == NULL || PyUnicode_READY(s) == -1) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
@@ -1475,11 +1477,8 @@
c = tok_nextc(tok);
}
tok_backup(tok, c);
- if (nonascii &&
- !verify_identifier(tok)) {
- tok->done = E_IDENTIFIER;
+ if (nonascii && !verify_identifier(tok))
return ERRORTOKEN;
- }
*p_start = tok->start;
*p_end = tok->cur;
return NAME;
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list