[Python-checkins] python/dist/src/Parser myreadline.c, 2.26.26.2, 2.26.26.3 parsetok.c, 2.30.2.3, 2.30.2.4 pgenmain.c, 2.26.2.2, 2.26.2.3 tokenizer.c, 2.54.2.2, 2.54.2.3

[jhylton@users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv27718/Parser

Modified Files:
      Tag: ast-branch
	myreadline.c parsetok.c pgenmain.c tokenizer.c 
Log Message:
Merge head to branch (for the last time)


Index: myreadline.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/myreadline.c,v
retrieving revision 2.26.26.2
retrieving revision 2.26.26.3
diff -u -d -r2.26.26.2 -r2.26.26.3
--- myreadline.c	7 Jan 2005 07:04:34 -0000	2.26.26.2
+++ myreadline.c	16 Oct 2005 05:24:05 -0000	2.26.26.3
@@ -82,9 +82,13 @@
 #ifdef EINTR
 		if (errno == EINTR) {
 			int s;
+#ifdef WITH_THREAD
 			PyEval_RestoreThread(_PyOS_ReadlineTState);
+#endif
 			s = PyErr_CheckSignals();
+#ifdef WITH_THREAD
 			PyEval_SaveThread();
+#endif
 			if (s < 0) {
 				return 1;
 			}

Index: parsetok.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/parsetok.c,v
retrieving revision 2.30.2.3
retrieving revision 2.30.2.4
diff -u -d -r2.30.2.3 -r2.30.2.4
--- parsetok.c	7 Jan 2005 07:04:35 -0000	2.30.2.3
+++ parsetok.c	16 Oct 2005 05:24:05 -0000	2.30.2.4
@@ -42,7 +42,7 @@
 	initerr(err_ret, filename);
 
 	if ((tok = PyTokenizer_FromString(s)) == NULL) {
-		err_ret->error = E_NOMEM;
+		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
 		return NULL;
 	}
 

Index: pgenmain.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/pgenmain.c,v
retrieving revision 2.26.2.2
retrieving revision 2.26.2.3
diff -u -d -r2.26.2.2 -r2.26.2.3
--- pgenmain.c	7 Jan 2005 07:04:35 -0000	2.26.2.2
+++ pgenmain.c	16 Oct 2005 05:24:05 -0000	2.26.2.3
@@ -116,6 +116,13 @@
 	return g;
 }
 
+/* Can't happen in pgen */
+PyObject*
+PyErr_Occurred()
+{
+	return 0;
+}
+
 void
 Py_FatalError(const char *msg)
 {

Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.54.2.2
retrieving revision 2.54.2.3
diff -u -d -r2.54.2.2 -r2.54.2.3
--- tokenizer.c	7 Jan 2005 07:04:35 -0000	2.54.2.2
+++ tokenizer.c	16 Oct 2005 05:24:05 -0000	2.54.2.3
@@ -334,7 +334,19 @@
 }
 
 /* Read a line of text from TOK into S, using the stream in TOK.
-   Return NULL on failure, else S.  */
+   Return NULL on failure, else S.
+   
+   On entry, tok->decoding_buffer will be one of:
+     1) NULL: need to call tok->decoding_readline to get a new line
+     2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
+           stored the result in tok->decoding_buffer
+     3) PyStringObject *: previous call to fp_readl did not have enough room
+           (in the s buffer) to copy entire contents of the line read
+           by tok->decoding_readline.  tok->decoding_buffer has the overflow.
+           In this case, fp_readl is called in a loop (with an expanded buffer)
+           until the buffer ends with a '\n' (or until the end of the file is 
+           reached): see tok_nextc and its calls to decoding_fgets.
+*/
 
 static char *
 fp_readl(char *s, int size, struct tok_state *tok)
@@ -344,32 +356,45 @@
 	Py_FatalError("fp_readl should not be called in this build.");
 	return NULL; /* Keep compiler happy (not reachable) */
 #else
-	PyObject* utf8;
+	PyObject* utf8 = NULL;
 	PyObject* buf = tok->decoding_buffer;
+	char *str;
+	int utf8len;
+
+	/* Ask for one less byte so we can terminate it */
+	assert(size > 0);
+	size--;
+
 	if (buf == NULL) {
-		/* Ask for one less byte so we can terminate it */
-		PyObject *args = Py_BuildValue("(i)", size-1);
-		if (args == NULL)
-			return error_ret(tok);
-		buf = PyObject_Call(tok->decoding_readline, args, NULL);
-		Py_DECREF(args);
+		buf = PyObject_CallObject(tok->decoding_readline, NULL);
 		if (buf == NULL)
 			return error_ret(tok);
 	} else {
 		tok->decoding_buffer = NULL;
+		if (PyString_CheckExact(buf))
+			utf8 = buf;
 	}
-	utf8 = PyUnicode_AsUTF8String(buf);
-	Py_DECREF(buf);
-	if (utf8 == NULL)
-		return error_ret(tok);
-	else {
-		const char* str = PyString_AsString(utf8);
-		assert(strlen(str) < (size_t)size); /* XXX */
-		strcpy(s, str);
-		Py_DECREF(utf8);
-		if (s[0] == '\0') return NULL; /* EOF */
-		return s;
+	if (utf8 == NULL) {
+		utf8 = PyUnicode_AsUTF8String(buf);
+		Py_DECREF(buf);
+		if (utf8 == NULL)
+			return error_ret(tok);
+	}
+	str = PyString_AsString(utf8);
+	utf8len = PyString_GET_SIZE(utf8);
+	if (utf8len > size) {
+		tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
+		if (tok->decoding_buffer == NULL) {
+			Py_DECREF(utf8);
+			return error_ret(tok);
+		}
+		utf8len = size;
 	}
+	memcpy(s, str, utf8len);
+	s[utf8len] = '\0';
+	Py_DECREF(utf8);
+	if (utf8len == 0) return NULL; /* EOF */
+	return s;
 #endif
 }
 
@@ -491,14 +516,7 @@
 	} else {
 		PyObject* buf = tok->decoding_buffer;
 		if (buf == NULL) {
-			PyObject *args = PyTuple_New(0);
-			if (args == NULL) {
-				error_ret(tok);
-				return 1;
-			}
-			buf = PyObject_Call(tok->decoding_readline,
-					    args, NULL);
-			Py_DECREF(args);
+			buf = PyObject_CallObject(tok->decoding_readline, NULL);
 			if (buf == NULL) {
 				error_ret(tok);
 				return 1;
@@ -585,8 +603,11 @@
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);
 		utf8 = translate_into_utf8(str, tok->enc);
-		if (utf8 == NULL)
+		if (utf8 == NULL) {
+			PyErr_Format(PyExc_SyntaxError,
+				"unknown encoding: %s", tok->enc);
 			return NULL;
+		}
 		str = PyString_AsString(utf8);
 	}
 #endif
@@ -1413,7 +1434,7 @@
 	if (c == '\\') {
 		c = tok_nextc(tok);
 		if (c != '\n') {
-			tok->done = E_TOKEN;
+			tok->done = E_LINECONT;
 			tok->cur = tok->inp;
 			return ERRORTOKEN;
 		}