https://github.com/python/cpython/commit/a4dfe1c9eaca6faf206f63a178233ffea20... commit: a4dfe1c9eaca6faf206f63a178233ffea208c906 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: Serhiy Storchaka <storchaka@gmail.com> date: 2017-09-30T22:51:37+03:00 summary: [3.6] bpo-31592: Fix an assertion failure in Python parser in case of a bad unicodedata.normalize(). (GH-3767) (#3836) (cherry picked from commit 7dc46d8cf5854d9f4ce3271b29c21aea4872e8ad) files: A Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst M Lib/test/test_ast.py M Python/ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8c62408bd8a..e68d0de2f8b 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -421,6 +421,16 @@ def test_empty_yield_from(self): compile(empty_yield_from, "<test>", "exec") self.assertIn("field value is required", str(cm.exception)) + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + class ASTHelpers_Test(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst new file mode 100644 index 00000000000..29f3461921e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst @@ -0,0 +1,2 @@ +Fixed an assertion failure in Python parser in case of a bad `unicodedata.normalize()`. +Patch by Oren Milman. diff --git a/Python/ast.c b/Python/ast.c index aa4acc9b261..2973b9fd866 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -589,7 +589,6 @@ struct compiling { PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ - PyObject *c_normalize_args; /* Normalization argument tuple. */ }; static asdl_seq *seq_for_testlist(struct compiling *, const node *); @@ -624,12 +623,6 @@ init_normalization(struct compiling *c) Py_DECREF(m); if (!c->c_normalize) return 0; - c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None); - if (!c->c_normalize_args) { - Py_CLEAR(c->c_normalize); - return 0; - } - PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL); return 1; } @@ -645,15 +638,29 @@ new_identifier(const char *n, struct compiling *c) identifier; if so, normalize to NFKC. */ if (!PyUnicode_IS_ASCII(id)) { PyObject *id2; + _Py_IDENTIFIER(NFKC); if (!c->c_normalize && !init_normalization(c)) { Py_DECREF(id); return NULL; } - PyTuple_SET_ITEM(c->c_normalize_args, 1, id); - id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL); + PyObject *form = _PyUnicode_FromId(&PyId_NFKC); + if (form == NULL) { + Py_DECREF(id); + return NULL; + } + PyObject *args[2] = {form, id}; + id2 = _PyObject_FastCall(c->c_normalize, args, 2); Py_DECREF(id); if (!id2) return NULL; + if (!PyUnicode_Check(id2)) { + PyErr_Format(PyExc_TypeError, + "unicodedata.normalize() must return a string, not " + "%.200s", + Py_TYPE(id2)->tp_name); + Py_DECREF(id2); + return NULL; + } id = id2; } PyUnicode_InternInPlace(&id); @@ -773,7 +780,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, /* borrowed reference */ c.c_filename = filename; c.c_normalize = NULL; - c.c_normalize_args = NULL; if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -866,8 +872,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, out: if (c.c_normalize) { Py_DECREF(c.c_normalize); - PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL); - Py_DECREF(c.c_normalize_args); } return res; }