[Python-checkins] r84642 - in python/branches/py3k: Doc/library/tokenize.rst Lib/tokenize.py

[raymond.hettinger]

Author: raymond.hettinger
Date: Thu Sep  9 06:32:39 2010
New Revision: 84642

Log:
Improve the repr for the TokenInfo named tuple.

Modified:
   python/branches/py3k/Doc/library/tokenize.rst
   python/branches/py3k/Lib/tokenize.py

Modified: python/branches/py3k/Doc/library/tokenize.rst
==============================================================================
--- python/branches/py3k/Doc/library/tokenize.rst	(original)
+++ python/branches/py3k/Doc/library/tokenize.rst	Thu Sep  9 06:32:39 2010
@@ -111,7 +111,7 @@
                 return fp.read()
 
 
-Example of a script re-writer that transforms float literals into Decimal
+Example of a script rewriter that transforms float literals into Decimal
 objects::
 
     from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP

Modified: python/branches/py3k/Lib/tokenize.py
==============================================================================
--- python/branches/py3k/Lib/tokenize.py	(original)
+++ python/branches/py3k/Lib/tokenize.py	Thu Sep  9 06:32:39 2010
@@ -63,7 +63,9 @@
         return result
 
     def __repr__(self):
-        return 'TokenInfo(type=%r, string=%r, start=%r, end=%r, line=%r)' % self
+        typ = self[0]
+        return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \
+               ((('%d (%s)' % (typ, tok_name[typ])),) + self[1:])
 
     def _asdict(self):
         'Return a new dict which maps field names to their values'
@@ -550,3 +552,28 @@
 # library that expect to be able to use tokenize with strings
 def generate_tokens(readline):
     return _tokenize(readline, None)
+
+if __name__ == "__main__":
+    # Quick sanity check
+    s = b'''def parseline(self, line):
+            """Parse the line into a command name and a string containing
+            the arguments.  Returns a tuple containing (command, args, line).
+            'command' and 'args' may be None if the line couldn't be parsed.
+            """
+            line = line.strip()
+            if not line:
+                return None, None, line
+            elif line[0] == '?':
+                line = 'help ' + line[1:]
+            elif line[0] == '!':
+                if hasattr(self, 'do_shell'):
+                    line = 'shell ' + line[1:]
+                else:
+                    return None, None, line
+            i, n = 0, len(line)
+            while i < n and line[i] in self.identchars: i = i+1
+            cmd, arg = line[:i], line[i:].strip()
+            return cmd, arg, line
+    '''
+    for tok in tokenize(iter(s.splitlines()).__next__):
+        print(tok)