[pypy-commit] pypy default: rlib parsing token_class extended with token_position_class

[xando]

Author: Sebastian Pawlu? <sebastian.pawlus at gmail.com>
Branch: 
Changeset: r71040:01046366784a
Date: 2014-04-28 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/01046366784a/

Log:	rlib parsing token_class extended with token_position_class

diff --git a/rpython/rlib/parsing/lexer.py b/rpython/rlib/parsing/lexer.py
--- a/rpython/rlib/parsing/lexer.py
+++ b/rpython/rlib/parsing/lexer.py
@@ -107,7 +107,7 @@
         self.matcher = matcher
         self.lineno = 0
         self.columnno = 0
-        
+
     def find_next_token(self):
         while 1:
             self.state = 0
@@ -126,8 +126,8 @@
                 i = ~i
                 stop = self.last_matched_index + 1
                 assert stop >= 0
-                if start == stop:   
-                    source_pos = SourcePos(i - 1, self.lineno, self.columnno)
+                if start == stop:
+                    source_pos = self.token_position_class(i - 1, self.lineno, self.columnno)
                     raise deterministic.LexerError(self.text, self.state,
                                                    source_pos)
                 source = self.text[start:stop]
@@ -147,7 +147,7 @@
                     else:
                         raise StopIteration
                 return result
-            source_pos = SourcePos(i - 1, self.lineno, self.columnno)
+            source_pos = self.token_position_class(i - 1, self.lineno, self.columnno)
             raise deterministic.LexerError(self.text, self.state, source_pos)
 
     def adjust_position(self, token):
@@ -158,7 +158,7 @@
             self.columnno += len(token)
         else:
             self.columnno = token.rfind("\n")
-    
+
 #    def inner_loop(self, i):
 #        while i < len(self.text):
 #            char = self.text[i]
@@ -186,10 +186,15 @@
 class LexingDFARunner(AbstractLexingDFARunner):
     def __init__(self, matcher, automaton, text, ignore, eof=False,
                  token_class=None):
-        if token_class is None:
+
+        if not token_class:
             self.token_class = Token
+            self.token_position_class = SourcePos
+
         else:
             self.token_class = token_class
+            self.token_position_class = token_class.source_position_class
+
         AbstractLexingDFARunner.__init__(self, matcher, automaton, text, eof)
         self.ignore = ignore
 
@@ -198,7 +203,8 @@
 
     def make_token(self, index, state, text, eof=False):
         assert (eof and state == -1) or 0 <= state < len(self.automaton.names)
-        source_pos = SourcePos(index, self.lineno, self.columnno)
+
+        source_pos = self.token_position_class(index, self.lineno, self.columnno)
         if eof:
             return self.token_class("EOF", "EOF", source_pos)
         return self.token_class(self.automaton.names[self.last_matched_state],