[Python-checkins] python/dist/src/Parser tokenizer.c,2.58,2.59 tokenizer.h,2.17,2.18

[loewis@users.sourceforge.net]

Update of /cvsroot/python/python/dist/src/Parser
In directory usw-pr-cvs1:/tmp/cvs-serv6376/Parser

Modified Files:
	tokenizer.c tokenizer.h 
Log Message:
Make pgen compile with pydebug. Duplicate normalized names, as it may
be longer than the old string.


Index: tokenizer.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.c,v
retrieving revision 2.58
retrieving revision 2.59
diff -C2 -d -r2.58 -r2.59
*** tokenizer.c	4 Aug 2002 18:28:44 -0000	2.58
--- tokenizer.c	4 Aug 2002 20:10:29 -0000	2.59
***************
*** 129,134 ****
--- 129,136 ----
  	tok->issued_encoding_warning = 0;
  	tok->encoding = NULL;
+ #ifndef PGEN
  	tok->decoding_readline = NULL;
  	tok->decoding_buffer = NULL;
+ #endif
  	return tok;
  }
***************
*** 226,231 ****
  				char* q = get_normal_name(r);
  				if (r != q) {
! 					assert(strlen(r) >= strlen(q));
! 					strcpy(r, q);
  				}
  				return r;
--- 228,233 ----
  				char* q = get_normal_name(r);
  				if (r != q) {
! 					PyMem_DEL(r);
! 					r = new_string(q, strlen(q));
  				}
  				return r;
***************
*** 585,590 ****
--- 587,594 ----
  	if (tok->encoding != NULL)
  		PyMem_DEL(tok->encoding);
+ #ifndef PGEN
  	Py_XDECREF(tok->decoding_readline);
  	Py_XDECREF(tok->decoding_buffer);
+ #endif
  	if (tok->fp != NULL && tok->buf != NULL)
  		PyMem_DEL(tok->buf);

Index: tokenizer.h
===================================================================
RCS file: /cvsroot/python/python/dist/src/Parser/tokenizer.h,v
retrieving revision 2.17
retrieving revision 2.18
diff -C2 -d -r2.17 -r2.18
*** tokenizer.h	4 Aug 2002 17:29:52 -0000	2.17
--- tokenizer.h	4 Aug 2002 20:10:29 -0000	2.18
***************
*** 46,51 ****
--- 46,53 ----
  	int issued_encoding_warning; /* whether non-ASCII warning was issued */
  	char *encoding;
+ #ifndef PGEN
  	PyObject *decoding_readline; /* codecs.open(...).readline */
  	PyObject *decoding_buffer;
+ #endif
  	const char* enc;
  	const char* str;