[Python-checkins] r51515 - python/trunk/Lib/test/test_tokenize.py
jeremy.hylton
python-checkins at python.org
Wed Aug 23 20:37:43 CEST 2006
Author: jeremy.hylton
Date: Wed Aug 23 20:37:43 2006
New Revision: 51515
Modified:
python/trunk/Lib/test/test_tokenize.py
Log:
Baby steps towards better tests for tokenize
Modified: python/trunk/Lib/test/test_tokenize.py
==============================================================================
--- python/trunk/Lib/test/test_tokenize.py (original)
+++ python/trunk/Lib/test/test_tokenize.py Wed Aug 23 20:37:43 2006
@@ -1,9 +1,36 @@
+"""Tests for the tokenize module.
+
+The tests were originally written in the old Python style, where the
+test output was compared to a golden file. This docstring represents
+the first steps towards rewriting the entire test as a doctest.
+
+The tests can be really simple. Given a small fragment of source
+code, print out a table with the tokens. The ENDMARK is omitted for
+brevity.
+
+>>> dump_tokens("1 + 1")
+NUMBER '1' (1, 0) (1, 1)
+OP '+' (1, 2) (1, 3)
+NUMBER '1' (1, 4) (1, 5)
+
+There will be a bunch more tests of specific source patterns.
+
+The tokenize module also defines an untokenize function that should
+regenerate the original program text from the tokens. (It doesn't
+work very well at the moment.)
+
+>>> roundtrip("if x == 1:\\n"
+... " print x\\n")
+if x ==1 :
+ print x
+"""
+
import os, glob, random
from cStringIO import StringIO
from test.test_support import (verbose, findfile, is_resource_enabled,
TestFailed)
-from tokenize import (tokenize, generate_tokens, untokenize,
- NUMBER, NAME, OP, STRING)
+from tokenize import (tokenize, generate_tokens, untokenize, tok_name,
+ ENDMARKER, NUMBER, NAME, OP, STRING)
# Test roundtrip for `untokenize`. `f` is a file path. The source code in f
# is tokenized, converted back to source code via tokenize.untokenize(),
@@ -24,6 +51,22 @@
if t1 != t2:
raise TestFailed("untokenize() roundtrip failed for %r" % f)
+def dump_tokens(s):
+ """Print out the tokens in s in a table format.
+
+ The ENDMARKER is omitted.
+ """
+ f = StringIO(s)
+ for type, token, start, end, line in generate_tokens(f.readline):
+ if type == ENDMARKER:
+ break
+ type = tok_name[type]
+ print "%(type)-10.10s %(token)-10.10r %(start)s %(end)s" % locals()
+
+def roundtrip(s):
+ f = StringIO(s)
+ print untokenize(generate_tokens(f.readline)),
+
# This is an example from the docs, set up as a doctest.
def decistmt(s):
"""Substitute Decimals for floats in a string of statements.
@@ -105,7 +148,7 @@
# Run the doctests in this module.
from test import test_tokenize # i.e., this module
from test.test_support import run_doctest
- run_doctest(test_tokenize)
+ run_doctest(test_tokenize, verbose)
if verbose:
print 'finished'
More information about the Python-checkins
mailing list