[Python-3000-checkins] r61462 - in python/branches/py3k: Lib/test/test_pep263.py Misc/NEWS Parser/parsetok.c Parser/tokenizer.c
martin.v.loewis
python-3000-checkins at python.org
Mon Mar 17 21:43:42 CET 2008
Author: martin.v.loewis
Date: Mon Mar 17 21:43:42 2008
New Revision: 61462
Modified:
python/branches/py3k/Lib/test/test_pep263.py
python/branches/py3k/Misc/NEWS
python/branches/py3k/Parser/parsetok.c
python/branches/py3k/Parser/tokenizer.c
Log:
Bug #2301: Don't try decoding the source code into the original
encoding for syntax errors.
Modified: python/branches/py3k/Lib/test/test_pep263.py
==============================================================================
Binary files. No diff available.
Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Mon Mar 17 21:43:42 2008
@@ -9,6 +9,12 @@
*Release date: XX-XXX-2008*
+Core and Builtins
+-----------------
+
+- Bug #2301: Don't try decoding the source code into the original
+ encoding for syntax errors.
+
Extension Modules
-----------------
Modified: python/branches/py3k/Parser/parsetok.c
==============================================================================
--- python/branches/py3k/Parser/parsetok.c (original)
+++ python/branches/py3k/Parser/parsetok.c Mon Mar 17 21:43:42 2008
@@ -213,21 +213,16 @@
err_ret->error = E_EOF;
err_ret->lineno = tok->lineno;
if (tok->buf != NULL) {
- char *text = NULL;
size_t len;
assert(tok->cur - tok->buf < INT_MAX);
err_ret->offset = (int)(tok->cur - tok->buf);
len = tok->inp - tok->buf;
- text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);
- if (text == NULL) {
- text = (char *) PyObject_MALLOC(len + 1);
- if (text != NULL) {
- if (len > 0)
- strncpy(text, tok->buf, len);
- text[len] = '\0';
- }
+ err_ret->text = (char *) PyObject_MALLOC(len + 1);
+ if (err_ret->text != NULL) {
+ if (len > 0)
+ strncpy(err_ret->text, tok->buf, len);
+ err_ret->text[len] = '\0';
}
- err_ret->text = text;
}
} else if (tok->encoding != NULL) {
node* r = PyNode_New(encoding_decl);
Modified: python/branches/py3k/Parser/tokenizer.c
==============================================================================
--- python/branches/py3k/Parser/tokenizer.c (original)
+++ python/branches/py3k/Parser/tokenizer.c Mon Mar 17 21:43:42 2008
@@ -1579,70 +1579,6 @@
return result;
}
-/* This function is only called from parsetok. However, it cannot live
- there, as it must be empty for PGEN, and we can check for PGEN only
- in this file. */
-
-#ifdef PGEN
-char*
-PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
-{
- return NULL;
-}
-#else
-static PyObject *
-dec_utf8(const char *enc, const char *text, size_t len) {
- PyObject *ret = NULL;
- PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");
- if (unicode_text) {
- ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");
- Py_DECREF(unicode_text);
- }
- if (!ret) {
- PyErr_Clear();
- }
- else {
- assert(PyString_Check(ret));
- }
- return ret;
-}
-
-char *
-PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
-{
- char *text = NULL;
- if (tok->encoding) {
- /* convert source to original encondig */
- PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);
- if (lineobj != NULL) {
- int linelen = PyString_GET_SIZE(lineobj);
- const char *line = PyString_AS_STRING(lineobj);
- text = PyObject_MALLOC(linelen + 1);
- if (text != NULL && line != NULL) {
- if (linelen)
- strncpy(text, line, linelen);
- text[linelen] = '\0';
- }
- Py_DECREF(lineobj);
-
- /* adjust error offset */
- if (*offset > 1) {
- PyObject *offsetobj = dec_utf8(tok->encoding,
- tok->buf,
- *offset-1);
- if (offsetobj) {
- *offset = 1 + Py_SIZE(offsetobj);
- Py_DECREF(offsetobj);
- }
- }
-
- }
- }
- return text;
-
-}
-#endif
-
/* Get -*- encoding -*- from a Python file.
PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
More information about the Python-3000-checkins
mailing list