r51526 - in python/trunk/Lib: test/output/test_tokenize test/test_tokenize.py tokenize.py
Aug. 23, 2006
11:14 p.m.
Author: jeremy.hylton
Date: Wed Aug 23 23:14:03 2006
New Revision: 51526
Modified:
python/trunk/Lib/test/output/test_tokenize
python/trunk/Lib/test/test_tokenize.py
python/trunk/Lib/tokenize.py
Log:
Bug fixes large and small for tokenize.
Small: Always generate a NL or NEWLINE token following
a COMMENT token. The old code did not generate an NL token if
the comment was on a line by itself.
Large: The output of untokenize() will now match the
input exactly if it is passed the full token sequence. The
old, crufty output is still generated if a limited input
sequence is provided, where limited means that it does not
include position information for tokens.
Remaining bug: There is no CONTINUATION token (\) so there is no way
for untokenize() to handle such code.
Also, expanded the number of doctests in hopes of eventually removing
the old-style tests that compare against a golden file.
Bug fix candidate for Python 2.5.1. (Sigh.)
Modified: python/trunk/Lib/test/output/test_tokenize
==============================================================================
--- python/trunk/Lib/test/output/test_tokenize (original)
+++ python/trunk/Lib/test/output/test_tokenize Wed Aug 23 23:14:03 2006
@@ -1,15 +1,23 @@
test_tokenize
-1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\n"
-2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \n'
+1,0-1,34: COMMENT "# Tests for the 'tokenize' module."
+1,34-1,35: NL '\n'
+2,0-2,42: COMMENT '# Large bits stolen from test_grammar.py. '
+2,42-2,43: NL '\n'
3,0-3,1: NL '\n'
-4,0-4,11: COMMENT '# Comments\n'
+4,0-4,10: COMMENT '# Comments'
+4,10-4,11: NL '\n'
5,0-5,3: STRING '"#"'
5,3-5,4: NEWLINE '\n'
-6,0-6,3: COMMENT "#'\n"
-7,0-7,3: COMMENT '#"\n'
-8,0-8,3: COMMENT '#\\\n'
-9,7-9,9: COMMENT '#\n'
-10,4-10,10: COMMENT '# abc\n'
+6,0-6,2: COMMENT "#'"
+6,2-6,3: NL '\n'
+7,0-7,2: COMMENT '#"'
+7,2-7,3: NL '\n'
+8,0-8,2: COMMENT '#\\'
+8,2-8,3: NL '\n'
+9,7-9,8: COMMENT '#'
+9,8-9,9: NL '\n'
+10,4-10,9: COMMENT '# abc'
+10,9-10,10: NL '\n'
11,0-12,4: STRING "'''#\n#'''"
12,4-12,5: NEWLINE '\n'
13,0-13,1: NL '\n'
@@ -19,7 +27,8 @@
14,7-14,8: COMMENT '#'
14,8-14,9: NEWLINE '\n'
15,0-15,1: NL '\n'
-16,0-16,25: COMMENT '# Balancing continuation\n'
+16,0-16,24: COMMENT '# Balancing continuation'
+16,24-16,25: NL '\n'
17,0-17,1: NL '\n'
18,0-18,1: NAME 'a'
18,2-18,3: OP '='
@@ -92,7 +101,8 @@
29,2-29,3: OP ')'
29,3-29,4: NEWLINE '\n'
30,0-30,1: NL '\n'
-31,0-31,37: COMMENT '# Backslash means line continuation:\n'
+31,0-31,36: COMMENT '# Backslash means line continuation:'
+31,36-31,37: NL '\n'
32,0-32,1: NAME 'x'
32,2-32,3: OP '='
32,4-32,5: NUMBER '1'
@@ -100,13 +110,15 @@
33,2-33,3: NUMBER '1'
33,3-33,4: NEWLINE '\n'
34,0-34,1: NL '\n'
-35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\n'
+35,0-35,54: COMMENT '# Backslash does not means continuation in comments :\\'
+35,54-35,55: NL '\n'
36,0-36,1: NAME 'x'
36,2-36,3: OP '='
36,4-36,5: NUMBER '0'
36,5-36,6: NEWLINE '\n'
37,0-37,1: NL '\n'
-38,0-38,20: COMMENT '# Ordinary integers\n'
+38,0-38,19: COMMENT '# Ordinary integers'
+38,19-38,20: NL '\n'
39,0-39,4: NUMBER '0xff'
39,5-39,7: OP '<>'
39,8-39,11: NUMBER '255'
@@ -137,7 +149,8 @@
44,15-44,16: NUMBER '1'
44,16-44,17: NEWLINE '\n'
45,0-45,1: NL '\n'
-46,0-46,16: COMMENT '# Long integers\n'
+46,0-46,15: COMMENT '# Long integers'
+46,15-46,16: NL '\n'
47,0-47,1: NAME 'x'
47,2-47,3: OP '='
47,4-47,6: NUMBER '0L'
@@ -171,7 +184,8 @@
54,4-54,35: NUMBER '123456789012345678901234567890l'
54,35-54,36: NEWLINE '\n'
55,0-55,1: NL '\n'
-56,0-56,25: COMMENT '# Floating-point numbers\n'
+56,0-56,24: COMMENT '# Floating-point numbers'
+56,24-56,25: NL '\n'
57,0-57,1: NAME 'x'
57,2-57,3: OP '='
57,4-57,8: NUMBER '3.14'
@@ -184,7 +198,8 @@
59,2-59,3: OP '='
59,4-59,9: NUMBER '0.314'
59,9-59,10: NEWLINE '\n'
-60,0-60,18: COMMENT '# XXX x = 000.314\n'
+60,0-60,17: COMMENT '# XXX x = 000.314'
+60,17-60,18: NL '\n'
61,0-61,1: NAME 'x'
61,2-61,3: OP '='
61,4-61,8: NUMBER '.314'
@@ -218,7 +233,8 @@
68,4-68,9: NUMBER '3.1e4'
68,9-68,10: NEWLINE '\n'
69,0-69,1: NL '\n'
-70,0-70,18: COMMENT '# String literals\n'
+70,0-70,17: COMMENT '# String literals'
+70,17-70,18: NL '\n'
71,0-71,1: NAME 'x'
71,2-71,3: OP '='
71,4-71,6: STRING "''"
@@ -366,7 +382,8 @@
125,6-126,3: STRING "uR'''spam\n'''"
126,3-126,4: NEWLINE '\n'
127,0-127,1: NL '\n'
-128,0-128,14: COMMENT '# Indentation\n'
+128,0-128,13: COMMENT '# Indentation'
+128,13-128,14: NL '\n'
129,0-129,2: NAME 'if'
129,3-129,4: NUMBER '1'
129,4-129,5: OP ':'
@@ -438,7 +455,8 @@
142,14-142,15: NUMBER '2'
142,15-142,16: NEWLINE '\n'
143,0-143,1: NL '\n'
-144,0-144,12: COMMENT '# Operators\n'
+144,0-144,11: COMMENT '# Operators'
+144,11-144,12: NL '\n'
145,0-145,1: NL '\n'
146,0-146,0: DEDENT ''
146,0-146,0: DEDENT ''
@@ -500,7 +518,8 @@
149,27-149,28: OP ')'
149,28-149,29: NEWLINE '\n'
150,0-150,1: NL '\n'
-151,0-151,13: COMMENT '# comparison\n'
+151,0-151,12: COMMENT '# comparison'
+151,12-151,13: NL '\n'
152,0-152,2: NAME 'if'
152,3-152,4: NUMBER '1'
152,5-152,6: OP '<'
@@ -531,7 +550,8 @@
152,67-152,71: NAME 'pass'
152,71-152,72: NEWLINE '\n'
153,0-153,1: NL '\n'
-154,0-154,9: COMMENT '# binary\n'
+154,0-154,8: COMMENT '# binary'
+154,8-154,9: NL '\n'
155,0-155,1: NAME 'x'
155,2-155,3: OP '='
155,4-155,5: NUMBER '1'
@@ -551,7 +571,8 @@
157,8-157,9: NUMBER '1'
157,9-157,10: NEWLINE '\n'
158,0-158,1: NL '\n'
-159,0-159,8: COMMENT '# shift\n'
+159,0-159,7: COMMENT '# shift'
+159,7-159,8: NL '\n'
160,0-160,1: NAME 'x'
160,2-160,3: OP '='
160,4-160,5: NUMBER '1'
@@ -561,7 +582,8 @@
160,14-160,15: NUMBER '1'
160,15-160,16: NEWLINE '\n'
161,0-161,1: NL '\n'
-162,0-162,11: COMMENT '# additive\n'
+162,0-162,10: COMMENT '# additive'
+162,10-162,11: NL '\n'
163,0-163,1: NAME 'x'
163,2-163,3: OP '='
163,4-163,5: NUMBER '1'
@@ -575,7 +597,8 @@
163,20-163,21: NUMBER '1'
163,21-163,22: NEWLINE '\n'
164,0-164,1: NL '\n'
-165,0-165,17: COMMENT '# multiplicative\n'
+165,0-165,16: COMMENT '# multiplicative'
+165,16-165,17: NL '\n'
166,0-166,1: NAME 'x'
166,2-166,3: OP '='
166,4-166,5: NUMBER '1'
@@ -587,7 +610,8 @@
166,16-166,17: NUMBER '1'
166,17-166,18: NEWLINE '\n'
167,0-167,1: NL '\n'
-168,0-168,8: COMMENT '# unary\n'
+168,0-168,7: COMMENT '# unary'
+168,7-168,8: NL '\n'
169,0-169,1: NAME 'x'
169,2-169,3: OP '='
169,4-169,5: OP '~'
@@ -625,7 +649,8 @@
170,24-170,25: NUMBER '1'
170,25-170,26: NEWLINE '\n'
171,0-171,1: NL '\n'
-172,0-172,11: COMMENT '# selector\n'
+172,0-172,10: COMMENT '# selector'
+172,10-172,11: NL '\n'
173,0-173,6: NAME 'import'
173,7-173,10: NAME 'sys'
173,10-173,11: OP ','
Modified: python/trunk/Lib/test/test_tokenize.py
==============================================================================
--- python/trunk/Lib/test/test_tokenize.py (original)
+++ python/trunk/Lib/test/test_tokenize.py Wed Aug 23 23:14:03 2006
@@ -9,20 +9,73 @@
brevity.
>>> dump_tokens("1 + 1")
-NUMBER '1' (1, 0) (1, 1)
-OP '+' (1, 2) (1, 3)
-NUMBER '1' (1, 4) (1, 5)
+NUMBER '1' (1, 0) (1, 1)
+OP '+' (1, 2) (1, 3)
+NUMBER '1' (1, 4) (1, 5)
+
+A comment generates a token here, unlike in the parser module. The
+comment token is followed by an NL or a NEWLINE token, depending on
+whether the line contains the completion of a statement.
+
+>>> dump_tokens("if False:\\n"
+... " # NL\\n"
+... " True = False # NEWLINE\\n")
+NAME 'if' (1, 0) (1, 2)
+NAME 'False' (1, 3) (1, 8)
+OP ':' (1, 8) (1, 9)
+NEWLINE '\\n' (1, 9) (1, 10)
+COMMENT '# NL' (2, 4) (2, 8)
+NL '\\n' (2, 8) (2, 9)
+INDENT ' ' (3, 0) (3, 4)
+NAME 'True' (3, 4) (3, 8)
+OP '=' (3, 9) (3, 10)
+NAME 'False' (3, 11) (3, 16)
+COMMENT '# NEWLINE' (3, 17) (3, 26)
+NEWLINE '\\n' (3, 26) (3, 27)
+DEDENT '' (4, 0) (4, 0)
+
There will be a bunch more tests of specific source patterns.
The tokenize module also defines an untokenize function that should
-regenerate the original program text from the tokens. (It doesn't
-work very well at the moment.)
+regenerate the original program text from the tokens.
+
+There are some standard formatting practices that are easy to get right.
>>> roundtrip("if x == 1:\\n"
... " print x\\n")
-if x ==1 :
- print x
+if x == 1:
+ print x
+
+Some people use different formatting conventions, which makes
+untokenize a little trickier. Note that this test involves trailing
+whitespace after the colon. You can't see it, but it's there!
+
+>>> roundtrip("if x == 1 : \\n"
+... " print x\\n")
+if x == 1 :
+ print x
+
+Comments need to go in the right place.
+
+>>> roundtrip("if x == 1:\\n"
+... " # A comment by itself.\\n"
+... " print x # Comment here, too.\\n"
+... " # Another comment.\\n"
+... "after_if = True\\n")
+if x == 1:
+ # A comment by itself.
+ print x # Comment here, too.
+ # Another comment.
+after_if = True
+
+>>> roundtrip("if (x # The comments need to go in the right place\\n"
+... " == 1):\\n"
+... " print 'x == 1'\\n")
+if (x # The comments need to go in the right place
+ == 1):
+ print 'x == 1'
+
"""
import os, glob, random
@@ -30,7 +83,7 @@
from test.test_support import (verbose, findfile, is_resource_enabled,
TestFailed)
from tokenize import (tokenize, generate_tokens, untokenize, tok_name,
- ENDMARKER, NUMBER, NAME, OP, STRING)
+ ENDMARKER, NUMBER, NAME, OP, STRING, COMMENT)
# Test roundtrip for `untokenize`. `f` is a file path. The source code in f
# is tokenized, converted back to source code via tokenize.untokenize(),
@@ -61,11 +114,12 @@
if type == ENDMARKER:
break
type = tok_name[type]
- print "%(type)-10.10s %(token)-10.10r %(start)s %(end)s" % locals()
+ print "%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals()
def roundtrip(s):
f = StringIO(s)
- print untokenize(generate_tokens(f.readline)),
+ source = untokenize(generate_tokens(f.readline))
+ print source,
# This is an example from the docs, set up as a doctest.
def decistmt(s):
Modified: python/trunk/Lib/tokenize.py
==============================================================================
--- python/trunk/Lib/tokenize.py (original)
+++ python/trunk/Lib/tokenize.py Wed Aug 23 23:14:03 2006
@@ -159,14 +159,76 @@
for token_info in generate_tokens(readline):
tokeneater(*token_info)
+class Untokenizer:
+
+ def __init__(self):
+ self.tokens = []
+ self.prev_row = 1
+ self.prev_col = 0
+
+ def add_whitespace(self, start):
+ row, col = start
+ while row > self.prev_row:
+ print row, "<", self.prev_row
+ self.tokens.append("\n")
+ self.prev_row += 1
+ col_offset = col - self.prev_col
+ if col_offset:
+ self.tokens.append(" " * col_offset)
+
+ def untokenize(self, iterable):
+ for t in iterable:
+ if len(t) == 2:
+ self.compat(t, iterable)
+ break
+ tok_type, token, start, end, line = t
+ self.add_whitespace(start)
+ self.tokens.append(token)
+ self.prev_row, self.prev_col = end
+ if tok_type in (NEWLINE, NL):
+ self.prev_row += 1
+ self.prev_col = 0
+ return "".join(self.tokens)
+
+ def compat(self, token, iterable):
+ startline = False
+ indents = []
+ toks_append = self.tokens.append
+ toknum, tokval = token
+ if toknum in (NAME, NUMBER):
+ tokval += ' '
+ if toknum in (NEWLINE, NL):
+ startline = True
+ for tok in iterable:
+ toknum, tokval = tok[:2]
+
+ if toknum in (NAME, NUMBER):
+ tokval += ' '
+
+ if toknum == INDENT:
+ indents.append(tokval)
+ continue
+ elif toknum == DEDENT:
+ indents.pop()
+ continue
+ elif toknum in (NEWLINE, NL):
+ startline = True
+ elif startline and indents:
+ toks_append(indents[-1])
+ startline = False
+ toks_append(tokval)
def untokenize(iterable):
"""Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
- with at least two elements, a token number and token value.
+ with at least two elements, a token number and token value. If
+ only two tokens are passed, the resulting output is poor.
+
+ Round-trip invariant for full input:
+ Untokenized source will match input source exactly
- Round-trip invariant:
+ Round-trip invariant for limited intput:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
@@ -174,31 +236,8 @@
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
"""
-
- startline = False
- indents = []
- toks = []
- toks_append = toks.append
- for tok in iterable:
- toknum, tokval = tok[:2]
-
- if toknum in (NAME, NUMBER):
- tokval += ' '
-
- if toknum == INDENT:
- indents.append(tokval)
- continue
- elif toknum == DEDENT:
- indents.pop()
- continue
- elif toknum in (NEWLINE, COMMENT, NL):
- startline = True
- elif startline and indents:
- toks_append(indents[-1])
- startline = False
- toks_append(tokval)
- return ''.join(toks)
-
+ ut = Untokenizer()
+ return ut.untokenize(iterable)
def generate_tokens(readline):
"""
@@ -237,7 +276,7 @@
if endmatch:
pos = end = endmatch.end(0)
yield (STRING, contstr + line[:end],
- strstart, (lnum, end), contline + line)
+ strstart, (lnum, end), contline + line)
contstr, needcont = '', 0
contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
@@ -263,7 +302,15 @@
if pos == max: break
if line[pos] in '#\r\n': # skip comments or blank lines
- yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
+ if line[pos] == '#':
+ comment_token = line[pos:].rstrip('\r\n')
+ nl_pos = pos + len(comment_token)
+ yield (COMMENT, comment_token,
+ (lnum, pos), (lnum, pos + len(comment_token)), line)
+ yield (NL, line[nl_pos:],
+ (lnum, nl_pos), (lnum, len(line)), line)
+ else:
+ yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line)
continue
@@ -294,9 +341,10 @@
(initial == '.' and token != '.'): # ordinary number
yield (NUMBER, token, spos, epos, line)
elif initial in '\r\n':
- yield (parenlev > 0 and NL or NEWLINE,
- token, spos, epos, line)
+ yield (NL if parenlev > 0 else NEWLINE,
+ token, spos, epos, line)
elif initial == '#':
+ assert not token.endswith("\n")
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
6733
Age (days ago)
6733
Last active (days ago)
0 comments
1 participants
participants (1)
-
jeremy.hylton