[pypy-commit] pypy py3k: detect inconsisten use of tab and spaces and raise an inter-level TabError in that case. We still need to convert it to applevel

[antocuni]

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r57392:475b47287f05
Date: 2012-09-19 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/475b47287f05/

Log:	detect inconsisten use of tab and spaces and raise an inter-level
	TabError in that case. We still need to convert it to applevel

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -28,6 +28,12 @@
 class IndentationError(SyntaxError):
     pass
 
+class TabError(IndentationError):
+    def __init__(self, lineno=0, offset=0, text=None, filename=None,
+                 lastlineno=0):
+        msg = "inconsistent use of tabs and spaces in indentation"
+        IndentationError.__init__(self, msg, lineno, offset, text, filename, lastlineno)
+
 class ASTError(Exception):
     def __init__(self, msg, ast_node ):
         self.msg = msg
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -57,6 +57,7 @@
     single_quoted[t] = t
 
 tabsize = 8
+alttabsize = 1
 
 # PYPY MODIFICATION: removed TokenError class as it's not needed here
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -1,8 +1,8 @@
 from pypy.interpreter.pyparser import automata
 from pypy.interpreter.pyparser.pygram import tokens
 from pypy.interpreter.pyparser.pytoken import python_opmap
-from pypy.interpreter.pyparser.error import TokenError, TokenIndentationError
-from pypy.interpreter.pyparser.pytokenize import tabsize, whiteSpaceDFA, \
+from pypy.interpreter.pyparser.error import TokenError, TokenIndentationError, TabError
+from pypy.interpreter.pyparser.pytokenize import tabsize, alttabsize, whiteSpaceDFA, \
     triple_quoted, endDFAs, single_quoted, pseudoDFA
 from pypy.interpreter.astcompiler import consts
 
@@ -77,6 +77,7 @@
     contstr, needcont = '', 0
     contline = None
     indents = [0]
+    altindents = [0]
     last_comment = ''
     parenlevstart = (0, 0, "")
 
@@ -123,11 +124,18 @@
         elif parenlev == 0 and not continued:  # new statement
             if not line: break
             column = 0
+            altcolumn = 0
             while pos < max:                   # measure leading whitespace
-                if line[pos] == ' ': column = column + 1
-                elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
-                elif line[pos] == '\f': column = 0
-                else: break
+                if line[pos] == ' ':
+                    column = column + 1
+                    altcolumn = altcolumn + 1
+                elif line[pos] == '\t':
+                    column = (column/tabsize + 1)*tabsize
+                    altcolumn = (altcolumn/alttabsize + 1)*alttabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
                 pos = pos + 1
             if pos == max: break
 
@@ -135,17 +143,27 @@
                 # skip comments or blank lines
                 continue
 
-            if column > indents[-1]:           # count indents or dedents
+            if column == indents[-1]:
+                if altcolumn != altindents[-1]:
+                    raise TabError(lnum, pos, line)
+            elif column > indents[-1]:           # count indents or dedents
+                if altcolumn <= altindents[-1]:
+                    raise TabError(lnum, pos, line)
                 indents.append(column)
+                altindents.append(altcolumn)
                 token_list.append((tokens.INDENT, line[:pos], lnum, 0, line))
                 last_comment = ''
-            while column < indents[-1]:
-                indents = indents[:-1]
-                token_list.append((tokens.DEDENT, '', lnum, pos, line))
-                last_comment = ''
-            if column != indents[-1]:
-                err = "unindent does not match any outer indentation level"
-                raise TokenIndentationError(err, line, lnum, 0, token_list)
+            else:
+                while column < indents[-1]:
+                    indents = indents[:-1]
+                    altindents = altindents[:-1]
+                    token_list.append((tokens.DEDENT, '', lnum, pos, line))
+                    last_comment = ''
+                if column != indents[-1]:
+                    err = "unindent does not match any outer indentation level"
+                    raise TokenIndentationError(err, line, lnum, 0, token_list)
+                if altcolumn != altindents[-1]:
+                    raise TabError(lnum, pos, line)
 
         else:                                  # continued statement
             if not line:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -2,7 +2,7 @@
 import py
 from pypy.interpreter.pyparser import pyparse
 from pypy.interpreter.pyparser.pygram import syms, tokens
-from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
+from pypy.interpreter.pyparser.error import SyntaxError, IndentationError, TabError
 from pypy.interpreter.astcompiler import consts
 
 
@@ -86,6 +86,18 @@
         assert exc.msg == "unindent does not match any outer indentation level"
         assert exc.lineno == 3
 
+    def test_taberror(self):
+        src = """
+if 1:
+        pass
+    \tpass
+"""
+        exc = py.test.raises(TabError, "self.parse(src)").value
+        assert exc.msg == "inconsistent use of tabs and spaces in indentation"
+        assert exc.lineno == 4
+        assert exc.offset == 5
+        assert exc.text == "    \tpass\n"
+
     def test_mac_newline(self):
         self.parse("this_is\ra_mac\rfile")