[pypy-svn] r14243 - pypy/dist/pypy/interpreter/pyparser
adim at codespeak.net
adim at codespeak.net
Mon Jul 4 20:49:11 CEST 2005
Author: adim
Date: Mon Jul 4 20:49:08 2005
New Revision: 14243
Modified:
pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
Log:
fixed some linenos management bugs
added the syntactically incorrect line in the TokenError
Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonlexer.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py Mon Jul 4 20:49:08 2005
@@ -2,6 +2,7 @@
it obeys the TokenSource interface defined for the grammar
analyser in grammar.py
"""
+import symbol
from grammar import TokenSource, Token
# Don't import string for that ...
@@ -72,10 +73,13 @@
class TokenError(SyntaxError):
"""Raised when EOF is found prematuerly"""
- def __init__(self, msg, strstart, token_stack):
+ def __init__(self, msg, line, strstart, token_stack):
self.lineno, self.offset = strstart
- self.text = "XXX"
- self.msg = "TokenError at pos (%d, %d)" % (self.lineno, self.offset)
+ self.text = line
+ self.errlabel = msg
+ self.msg = "TokenError at pos (%d, %d) in %r" % (self.lineno,
+ self.offset,
+ line)
self.token_stack = token_stack
def generate_tokens(lines):
@@ -119,7 +123,7 @@
if contstr: # continued string
if not line:
- raise TokenError("EOF in multi-line string", strstart,
+ raise TokenError("EOF in multi-line string", line, strstart,
token_list)
endmatch = endDFA.recognize(line)
if -1 != endmatch:
@@ -189,7 +193,7 @@
else: # continued statement
if not line:
- raise TokenError("EOF in multi-line statement", (lnum, 0), token_list)
+ raise TokenError("EOF in multi-line statement", line, (lnum, 0), token_list)
continued = 0
while pos < max:
@@ -283,17 +287,16 @@
# (lnum, pos), (lnum, pos+1), line))
pos = pos + 1
- last_comment = ''
+ lnum -= 1
for indent in indents[1:]: # pop remaining indent levels
tok = token_from_values(tokenmod.DEDENT, '')
token_list.append((tok, line, lnum, pos))
# token_list.append((DEDENT, '', (lnum, 0), (lnum, 0), ''))
- lnum += 1
- ## <XXX> adim
- token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
+ ## <XXX> adim: this can't be (only) that, can it ?
+ if token_list and token_list[-1] != symbol.file_input:
+ token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
## </XXX>
- lnum += 1
tok = token_from_values(tokenmod.ENDMARKER, '',)
token_list.append((tok, line, lnum, pos))
# token_list.append((ENDMARKER, '', (lnum, 0), (lnum, 0), ''))
@@ -326,6 +329,9 @@
"""Returns the current line being parsed"""
return self._current_line
+ def current_lineno(self):
+ return self._lineno
+
def context(self):
"""Returns an opaque context object for later restore"""
return self.stack_pos
@@ -368,7 +374,7 @@
NAMED_LIST = [tokenmod.OP, ]
def token_from_values(tok_type, tok_string):
- """XXX Compatibility layer between both parsers"""
+ """Compatibility layer between both parsers"""
if tok_type in NONE_LIST:
return Token(tokenmod.tok_name[tok_type], None)
if tok_type in NAMED_LIST:
More information about the Pypy-commit
mailing list