[Python-checkins] python/dist/src/Parser parsetok.c, 2.36, 2.37 pgenmain.c, 2.31, 2.32 tokenizer.c, 2.78, 2.79

[nnorwitz@users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8132/Parser

Modified Files:
	parsetok.c pgenmain.c tokenizer.c 
Log Message:
- Fix segfault with invalid coding.
- SF Bug #772896, unknown encoding results in MemoryError, which is not helpful

I will only backport the segfault fix.  I'll let Anthony decide if he wants
the other changes backported.  I will do the backport if asked.


Index: parsetok.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/parsetok.c,v
retrieving revision 2.36
retrieving revision 2.37
diff -u -d -r2.36 -r2.37
--- parsetok.c	8 Jul 2004 01:54:07 -0000	2.36
+++ parsetok.c	2 Oct 2005 01:48:51 -0000	2.37
@@ -42,7 +42,7 @@
 	initerr(err_ret, filename);
 
 	if ((tok = PyTokenizer_FromString(s)) == NULL) {
-		err_ret->error = E_NOMEM;
+		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
 		return NULL;
 	}
 

Index: pgenmain.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/pgenmain.c,v
retrieving revision 2.31
retrieving revision 2.32
diff -u -d -r2.31 -r2.32
--- pgenmain.c	7 Feb 2004 13:53:46 -0000	2.31
+++ pgenmain.c	2 Oct 2005 01:48:51 -0000	2.32
@@ -116,6 +116,13 @@
 	return g;
 }
 
+/* Can't happen in pgen */
+PyObject*
+PyErr_Occurred()
+{
+	return 0;
+}
+
 void
 Py_FatalError(const char *msg)
 {

Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.78
retrieving revision 2.79
diff -u -d -r2.78 -r2.79
--- tokenizer.c	12 Jul 2005 21:53:43 -0000	2.78
+++ tokenizer.c	2 Oct 2005 01:48:51 -0000	2.79
@@ -603,8 +603,11 @@
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);
 		utf8 = translate_into_utf8(str, tok->enc);
-		if (utf8 == NULL)
+		if (utf8 == NULL) {
+			PyErr_Format(PyExc_SyntaxError,
+				"unknown encoding: %s", tok->enc);
 			return NULL;
+		}
 		str = PyString_AsString(utf8);
 	}
 #endif