[Python-checkins] r60169 - in python/branches/release25-maint: Lib/test/test_pep263.py Parser/tokenizer.c

[georg.brandl]

Author: georg.brandl
Date: Mon Jan 21 19:35:52 2008
New Revision: 60169

Modified:
   python/branches/release25-maint/Lib/test/test_pep263.py
   python/branches/release25-maint/Parser/tokenizer.c
Log:
Issue #1882: when compiling code from a string, encoding cookies in the
second line of code were not always recognized correctly.
 (backport from rev. 60168)

Modified: python/branches/release25-maint/Lib/test/test_pep263.py
==============================================================================
Binary files. No diff available.

Modified: python/branches/release25-maint/Parser/tokenizer.c
==============================================================================
--- python/branches/release25-maint/Parser/tokenizer.c	(original)
+++ python/branches/release25-maint/Parser/tokenizer.c	Mon Jan 21 19:35:52 2008
@@ -585,6 +585,7 @@
 {
 	PyObject* utf8 = NULL;
 	const char *s;
+	char *newl[2] = {NULL, NULL};
 	int lineno = 0;
 	tok->enc = NULL;
 	tok->str = str;
@@ -603,13 +604,23 @@
 	for (s = str;; s++) {
 		if (*s == '\0') break;
 		else if (*s == '\n') {
+			newl[lineno] = s;
 			lineno++;
 			if (lineno == 2) break;
 		}
 	}
 	tok->enc = NULL;
-	if (!check_coding_spec(str, s - str, tok, buf_setreadl))
-		return error_ret(tok);
+	/* need to check line 1 and 2 separately since check_coding_spec
+	   assumes a single line as input */
+	if (newl[0]) {
+		if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
+			return error_ret(tok);
+		if (tok->enc == NULL && newl[1]) {
+			if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
+					       tok, buf_setreadl))
+				return error_ret(tok);
+		}
+	}
 #ifdef Py_USING_UNICODE
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);