[pypy-svn] r14319 - in pypy/branch/dist-2.4.1/pypy/interpreter: . pyparser

arigo at codespeak.net arigo at codespeak.net
Tue Jul 5 23:33:46 CEST 2005


Author: arigo
Date: Tue Jul  5 23:33:43 2005
New Revision: 14319

Modified:
   pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py
   pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py
   pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py
   pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py
Log:
Some hacks to make the interactive --pyparse work.  Don't look too closely.
XXX needs to have the test_samples pass again...


Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py	(original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py	Tue Jul  5 23:33:43 2005
@@ -184,7 +184,7 @@
         # XXX use 'flags'
         space = self.space
         try:
-            tuples = pypy_parse(source, mode, True)
+            tuples = pypy_parse(source, mode, True, flags)
         except ParseError, e:
             raise OperationError(space.w_SyntaxError,
                                  e.wrap_info(space, filename))

Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py	(original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py	Tue Jul  5 23:33:43 2005
@@ -3,6 +3,7 @@
 analyser in grammar.py
 """
 import symbol, sys
+from codeop import PyCF_DONT_IMPLY_DEDENT
 
 from pypy.interpreter.pyparser.grammar import TokenSource, Token
 from pypy.interpreter.pyparser.error import ParseError
@@ -79,7 +80,7 @@
         ParseError.__init__(self, msg, lineno, offset, line)
         self.token_stack = token_stack
 
-def generate_tokens(lines):
+def generate_tokens(lines, flags):
     """
     This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
     the original function is not RPYTHON (uses yield)
@@ -282,23 +283,25 @@
                 pos = pos + 1
 
     lnum -= 1
-    for indent in indents[1:]:                 # pop remaining indent levels
-        tok = token_from_values(tokenmod.DEDENT, '')
-        token_list.append((tok, line, lnum, pos))
-        
-    ## <XXX> adim: this can't be (only) that, can it ?
-    if token_list and token_list[-1] != symbol.file_input:
-        token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
-    ## </XXX>
+    if not (flags & PyCF_DONT_IMPLY_DEDENT):
+        if token_list and token_list[-1][0].name != 'NEWLINE':
+            token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
+        for indent in indents[1:]:                 # pop remaining indent levels
+            tok = token_from_values(tokenmod.DEDENT, '')
+            token_list.append((tok, line, lnum, pos))
+    if token_list and token_list[-1][0].name == 'NEWLINE':
+        token_list.pop()
+    token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
+
     tok = token_from_values(tokenmod.ENDMARKER, '',)
     token_list.append((tok, line, lnum, pos))
     return token_list, encoding
 
 class PythonSource(TokenSource):
     """This source uses Jonathan's tokenizer"""
-    def __init__(self, strings):
+    def __init__(self, strings, flags=0):
         # TokenSource.__init__(self)
-        tokens, encoding = generate_tokens(strings)
+        tokens, encoding = generate_tokens(strings, flags)
         self.token_stack = tokens
         self.encoding = encoding
         self._current_line = '' # the current line (as a string)
@@ -313,7 +316,7 @@
         tok, line, lnum, pos = self.token_stack[self.stack_pos]
         self.stack_pos += 1
         self._current_line = line
-        self._lineno = lnum
+        self._lineno = max(self._lineno, lnum)
         self._offset = pos
         return tok
 

Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py	Tue Jul  5 23:33:43 2005
@@ -15,6 +15,7 @@
 import os
 import grammar
 import symbol
+from codeop import PyCF_DONT_IMPLY_DEDENT
 
 class PythonParser(object):
     """Wrapper class for python grammar"""
@@ -24,19 +25,20 @@
         # Build first sets for each rule (including anonymous ones)
         grammar.build_first_sets(self.items)
 
-    def parse_source(self, textsrc, goal, builder=None):
+    def parse_source(self, textsrc, goal, builder=None, flags=0):
         """Parse a python source according to goal"""
         lines = [line + '\n' for line in textsrc.split('\n')]
-        if textsrc == '\n':
+        if textsrc.endswith('\n'):
             lines.pop()
+            flags &= ~PyCF_DONT_IMPLY_DEDENT
         else:
             last_line = lines[-1]
             lines[-1] = last_line[:-1]
-        return self.parse_lines(lines, goal, builder)
+        return self.parse_lines(lines, goal, builder, flags)
 
-    def parse_lines(self, lines, goal, builder=None):
+    def parse_lines(self, lines, goal, builder=None, flags=0):
         target = self.rules[goal]
-        src = Source(lines)
+        src = Source(lines, flags)
         
         if builder is None:
             builder = grammar.BaseGrammarBuilder(debug=False, rules=self.rules)

Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py	(original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py	Tue Jul  5 23:33:43 2005
@@ -50,7 +50,7 @@
     pyf.close()
     return pypy_parse(source, 'exec', lineno)
 
-def pypy_parse(source, mode='exec', lineno=False):
+def pypy_parse(source, mode='exec', lineno=False, flags=0):
     """parse <source> using PyPy's parser module and return
     a tuple of three elements :
      - The encoding declaration symbol or None if there were no encoding
@@ -63,7 +63,7 @@
     """
     builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
     target_rule = TARGET_DICT[mode]
-    PYTHON_PARSER.parse_source(source, target_rule, builder)
+    PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
     stack_element = builder.stack[-1]
     # convert the stack element into nested tuples (caution, the annotator
     # can't follow this call)



More information about the Pypy-commit mailing list