[Python-3000-checkins] r55866 - python/branches/p3yk/Parser/tokenizer.c

[georg.brandl]

Author: georg.brandl
Date: Mon Jun 11 00:37:43 2007
New Revision: 55866

Modified:
   python/branches/p3yk/Parser/tokenizer.c
Log:
Tokenizer changes for PEP 3127.


Modified: python/branches/p3yk/Parser/tokenizer.c
==============================================================================
--- python/branches/p3yk/Parser/tokenizer.c	(original)
+++ python/branches/p3yk/Parser/tokenizer.c	Mon Jun 11 00:37:43 2007
@@ -1310,7 +1310,7 @@
 	/* Number */
 	if (isdigit(c)) {
 		if (c == '0') {
-			/* Hex or octal -- maybe. */
+			/* Hex, octal or binary -- maybe. */
 			c = tok_nextc(tok);
 			if (c == '.')
 				goto fraction;
@@ -1324,18 +1324,27 @@
 					c = tok_nextc(tok);
 				} while (isxdigit(c));
 			}
+                        else if (c == 'o' || c == 'O') {
+				/* Octal */
+				do {
+					c = tok_nextc(tok);
+				} while ('0' <= c && c < '8');
+			}
+			else if (c == 'b' || c == 'B') {
+				/* Binary */
+				do {
+					c = tok_nextc(tok);
+				} while (c == '0' || c == '1');
+			}
 			else {
-				int found_decimal = 0;
-				/* Octal; c is first char of it */
-				/* There's no 'isoctdigit' macro, sigh */
-				while ('0' <= c && c < '8') {
+				int nonzero = 0;
+				/* maybe old-style octal; c is first char of it */
+				/* in any case, allow '0' as a literal */
+				while (c == '0')
+					c = tok_nextc(tok);
+				while (isdigit(c)) {
+					nonzero = 1;
 					c = tok_nextc(tok);
-				}
-				if (isdigit(c)) {
-					found_decimal = 1;
-					do {
-						c = tok_nextc(tok);
-					} while (isdigit(c));
 				}
 				if (c == '.')
 					goto fraction;
@@ -1345,7 +1354,7 @@
 				else if (c == 'j' || c == 'J')
 					goto imaginary;
 #endif
-				else if (found_decimal) {
+				else if (nonzero) {
 					tok->done = E_TOKEN;
 					tok_backup(tok, c);
 					return ERRORTOKEN;