[pypy-svn] r65651 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test

benjamin at codespeak.net benjamin at codespeak.net
Sun Jun 7 23:09:29 CEST 2009


Author: benjamin
Date: Sun Jun  7 23:09:29 2009
New Revision: 65651

Modified:
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/error.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
improve token errors a bit

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/error.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/error.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/error.py	Sun Jun  7 23:09:29 2009
@@ -38,5 +38,8 @@
         SyntaxError.__init__(self, msg, lineno, column, line)
         self.tokens = tokens
 
-class TokenIndentationError(TokenError):
-    pass
+class TokenIndentationError(IndentationError):
+
+    def __init__(self, msg, line, lineno, column, tokens):
+        SyntaxError.__init__(self, msg, lineno, column, line)
+        self.tokens = tokens

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py	Sun Jun  7 23:09:29 2009
@@ -83,7 +83,9 @@
     endDFA = automata.DFA([], [])
     # make the annotator happy
     line = ''
-    for line in textsrc.splitlines(True):
+    lines = textsrc.splitlines(True)
+    lines.append('')
+    for line in lines:
         lnum = lnum + 1
         pos, max = 0, len(line)
 
@@ -94,7 +96,7 @@
             endmatch = endDFA.recognize(line)
             if endmatch >= 0:
                 pos = end = endmatch
-                tok = (tokens.STRING, constr + line[:end], lnum, pos, line)
+                tok = (tokens.STRING, contstr + line[:end], lnum, pos, line)
                 token_list.append(tok)
                 last_comment = ''
                 # token_list.append((STRING, contstr + line[:end],
@@ -103,7 +105,7 @@
                 contline = None
             elif (needcont and not line.endswith('\\\n') and
                                not line.endswith('\\\r\n')):
-                tok = (tokens.ERRORTOKEN, constr + line, lnum, pos, line)
+                tok = (tokens.ERRORTOKEN, contstr + line, lnum, pos, line)
                 token_list.append(tok)
                 last_comment = ''
                 # token_list.append((ERRORTOKEN, contstr + line,

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	Sun Jun  7 23:09:29 2009
@@ -14,13 +14,22 @@
         tree = self.parser.parse_source("name = 32")
         assert self.parser.root is None
 
-    def test_error(self):
+    def test_syntax_error(self):
         parse = self.parser.parse_source
         exc = py.test.raises(SyntaxError, parse, "name another for").value
         assert exc.msg == "invalid syntax"
         assert exc.lineno == 1
         assert exc.offset == 16
         assert exc.text == "name another for"
+        exc = py.test.raises(SyntaxError, parse, "\"blah").value
+        assert exc.msg == "EOL while scanning single-quoted string"
+        exc = py.test.raises(SyntaxError, parse, "'''\n").value
+        assert exc.msg == "EOF while scanning triple-quoted string"
+        for input in ("())", "(()", "((", "))"):
+            py.test.raises(SyntaxError, parse, input)
+
+    def test_indentation_error(self):
+        parse = self.parser.parse_source
         input = """
 def f():
 pass"""
@@ -32,6 +41,9 @@
         input = "hi\n    indented"
         exc = py.test.raises(IndentationError, parse, input).value
         assert exc.msg == "unexpected indent"
+        input = "def f():\n    pass\n  next_stmt"
+        exc = py.test.raises(IndentationError, parse, input).value
+        assert exc.msg == "unindent does not match any outer indentation level"
 
     def test_mode(self):
         assert self.parser.parse_source("x = 43*54").type == syms.file_input



More information about the Pypy-commit mailing list