[Python-checkins] python/dist/src/Parser tokenizer.c,2.65,2.66 tokenizer.h,2.18,2.19

[loewis@users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory usw-pr-cvs1:/tmp/cvs-serv9883/Parser

Modified Files:
	tokenizer.c tokenizer.h 
Log Message:
Ignore encoding declarations inside strings. Fixes #603509.


Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.65
retrieving revision 2.66
diff -C2 -d -r2.65 -r2.66
*** tokenizer.c	16 Aug 2002 17:01:09 -0000	2.65
--- tokenizer.c	3 Sep 2002 11:52:44 -0000	2.66
***************
*** 129,132 ****
--- 129,133 ----
  	tok->issued_encoding_warning = 0;
  	tok->encoding = NULL;
+         tok->cont_line = 0;
  #ifndef PGEN
  	tok->decoding_readline = NULL;
***************
*** 208,212 ****
  {
  	int i;
! 	for (i = 0; i < size - 6; i++) { /* XXX inefficient search */
  		const char* t = s + i;
  		if (strncmp(t, "coding", 6) == 0) {
--- 209,221 ----
  {
  	int i;
! 	/* Coding spec must be in a comment, and that comment must be
!          * the only statement on the source code line. */
!         for (i = 0; i < size - 6; i++) {
! 		if (s[i] == '#')
! 			break;
! 		if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
! 			return NULL;
! 	}
! 	for (; i < size - 6; i++) { /* XXX inefficient search */
  		const char* t = s + i;
  		if (strncmp(t, "coding", 6) == 0) {
***************
*** 248,251 ****
--- 257,263 ----
  {
  	int r = 1;
+         if (tok->cont_line)
+ 		/* It's a continuation line, so it can't be a coding spec. */
+ 		return 1;
  	char* cs = get_coding_spec(line, size);
  	if (cs != NULL) {
***************
*** 1159,1162 ****
--- 1171,1175 ----
  		*p_start = tok->start;
  		*p_end = tok->cur - 1; /* Leave '\n' out of the string */
+                 tok->cont_line = 0;
  		return NEWLINE;
  	}
***************
*** 1293,1296 ****
--- 1306,1310 ----
  				}
  				tripcount = 0;
+                                 tok->cont_line = 1; /* multiline string. */
  			}
  			else if (c == EOF) {
***************
*** 1341,1344 ****
--- 1355,1359 ----
  			return ERRORTOKEN;
  		}
+                 tok->cont_line = 1;
  		goto again; /* Read next line */
  	}

Index: tokenizer.h
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.h,v
retrieving revision 2.18
retrieving revision 2.19
diff -C2 -d -r2.18 -r2.19
*** tokenizer.h	4 Aug 2002 20:10:29 -0000	2.18
--- tokenizer.h	3 Sep 2002 11:52:44 -0000	2.19
***************
*** 46,49 ****
--- 46,50 ----
  	int issued_encoding_warning; /* whether non-ASCII warning was issued */
  	char *encoding;
+ 	int cont_line;          /* whether we are in a continuation line. */
  #ifndef PGEN
  	PyObject *decoding_readline; /* codecs.open(...).readline */