[Python-checkins] r52870 - in sandbox/trunk/2to3: example.py pgen2/driver.py tokenize.py

[guido.van.rossum]

Author: guido.van.rossum
Date: Thu Nov 30 20:11:10 2006
New Revision: 52870

Modified:
   sandbox/trunk/2to3/example.py
   sandbox/trunk/2to3/pgen2/driver.py
   sandbox/trunk/2to3/tokenize.py
Log:
Tweak the prefix computation to handle alternative horizontal whitespace
(\t, \f) properly.
Change tokenize.py to return a single (NL, "\\\n") token for a backslash
continuation line.


Modified: sandbox/trunk/2to3/example.py
==============================================================================
--- sandbox/trunk/2to3/example.py	(original)
+++ sandbox/trunk/2to3/example.py	Thu Nov 30 20:11:10 2006
@@ -1,8 +1,13 @@
 #!/usr/bin/python
+	# comment indented by tab
 """Docstring."""
 
 d = {"x": 42}
 if d.has_key("x") or d.has_key("y"):
-    print d["x"]
+    print d["x"] + \
+          42
+
+def foo():
+	pass # body indented by tab
 
 # This is the last line.

Modified: sandbox/trunk/2to3/pgen2/driver.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/driver.py	(original)
+++ sandbox/trunk/2to3/pgen2/driver.py	Thu Nov 30 20:11:10 2006
@@ -32,7 +32,7 @@
             logger = logging.getLogger()
         self.logger = logger
         self.convert = convert
- 
+
     def parse_stream_raw(self, stream, debug=False):
         """Parse a stream and return the concrete syntax tree."""
         p = parse.Parser(self.grammar, self.convert)
@@ -51,7 +51,7 @@
                     lineno = s_lineno
                     column = 0
                 if column < s_column:
-                    prefix += " " * (s_column - column)
+                    prefix += line_text[column:s_column]
                     column = s_column
             if type in (tokenize.COMMENT, tokenize.NL):
                 prefix += value
@@ -63,7 +63,8 @@
             if type == token.OP:
                 type = grammar.opmap[value]
             if debug:
-                self.logger.debug("%s %r", token.tok_name[type], value)
+                self.logger.debug("%s %r (prefix=%r)",
+                                  token.tok_name[type], value, prefix)
             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")

Modified: sandbox/trunk/2to3/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/tokenize.py	(original)
+++ sandbox/trunk/2to3/tokenize.py	Thu Nov 30 20:11:10 2006
@@ -371,7 +371,7 @@
                     yield (NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     # This yield is new; needed for better idempotency:
-                    yield (NL, initial, spos, (spos[0], spos[1]+1), line)
+                    yield (NL, token, spos, (lnum, pos), line)
                     continued = 1
                 else:
                     if initial in '([{': parenlev = parenlev + 1