[pypy-svn] r14319 - in pypy/branch/dist-2.4.1/pypy/interpreter: . pyparser
arigo at codespeak.net
arigo at codespeak.net
Tue Jul 5 23:33:46 CEST 2005
Author: arigo
Date: Tue Jul 5 23:33:43 2005
New Revision: 14319
Modified:
pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py
pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py
pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py
pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py
Log:
Some hacks to make the interactive --pyparse work. Don't look too closely.
XXX needs to have the test_samples pass again...
Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py (original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pycompiler.py Tue Jul 5 23:33:43 2005
@@ -184,7 +184,7 @@
# XXX use 'flags'
space = self.space
try:
- tuples = pypy_parse(source, mode, True)
+ tuples = pypy_parse(source, mode, True, flags)
except ParseError, e:
raise OperationError(space.w_SyntaxError,
e.wrap_info(space, filename))
Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py (original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonlexer.py Tue Jul 5 23:33:43 2005
@@ -3,6 +3,7 @@
analyser in grammar.py
"""
import symbol, sys
+from codeop import PyCF_DONT_IMPLY_DEDENT
from pypy.interpreter.pyparser.grammar import TokenSource, Token
from pypy.interpreter.pyparser.error import ParseError
@@ -79,7 +80,7 @@
ParseError.__init__(self, msg, lineno, offset, line)
self.token_stack = token_stack
-def generate_tokens(lines):
+def generate_tokens(lines, flags):
"""
This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
the original function is not RPYTHON (uses yield)
@@ -282,23 +283,25 @@
pos = pos + 1
lnum -= 1
- for indent in indents[1:]: # pop remaining indent levels
- tok = token_from_values(tokenmod.DEDENT, '')
- token_list.append((tok, line, lnum, pos))
-
- ## <XXX> adim: this can't be (only) that, can it ?
- if token_list and token_list[-1] != symbol.file_input:
- token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
- ## </XXX>
+ if not (flags & PyCF_DONT_IMPLY_DEDENT):
+ if token_list and token_list[-1][0].name != 'NEWLINE':
+ token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
+ for indent in indents[1:]: # pop remaining indent levels
+ tok = token_from_values(tokenmod.DEDENT, '')
+ token_list.append((tok, line, lnum, pos))
+ if token_list and token_list[-1][0].name == 'NEWLINE':
+ token_list.pop()
+ token_list.append((Token('NEWLINE', ''), '\n', lnum, 0))
+
tok = token_from_values(tokenmod.ENDMARKER, '',)
token_list.append((tok, line, lnum, pos))
return token_list, encoding
class PythonSource(TokenSource):
"""This source uses Jonathan's tokenizer"""
- def __init__(self, strings):
+ def __init__(self, strings, flags=0):
# TokenSource.__init__(self)
- tokens, encoding = generate_tokens(strings)
+ tokens, encoding = generate_tokens(strings, flags)
self.token_stack = tokens
self.encoding = encoding
self._current_line = '' # the current line (as a string)
@@ -313,7 +316,7 @@
tok, line, lnum, pos = self.token_stack[self.stack_pos]
self.stack_pos += 1
self._current_line = line
- self._lineno = lnum
+ self._lineno = max(self._lineno, lnum)
self._offset = pos
return tok
Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py (original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonparse.py Tue Jul 5 23:33:43 2005
@@ -15,6 +15,7 @@
import os
import grammar
import symbol
+from codeop import PyCF_DONT_IMPLY_DEDENT
class PythonParser(object):
"""Wrapper class for python grammar"""
@@ -24,19 +25,20 @@
# Build first sets for each rule (including anonymous ones)
grammar.build_first_sets(self.items)
- def parse_source(self, textsrc, goal, builder=None):
+ def parse_source(self, textsrc, goal, builder=None, flags=0):
"""Parse a python source according to goal"""
lines = [line + '\n' for line in textsrc.split('\n')]
- if textsrc == '\n':
+ if textsrc.endswith('\n'):
lines.pop()
+ flags &= ~PyCF_DONT_IMPLY_DEDENT
else:
last_line = lines[-1]
lines[-1] = last_line[:-1]
- return self.parse_lines(lines, goal, builder)
+ return self.parse_lines(lines, goal, builder, flags)
- def parse_lines(self, lines, goal, builder=None):
+ def parse_lines(self, lines, goal, builder=None, flags=0):
target = self.rules[goal]
- src = Source(lines)
+ src = Source(lines, flags)
if builder is None:
builder = grammar.BaseGrammarBuilder(debug=False, rules=self.rules)
Modified: pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py
==============================================================================
--- pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py (original)
+++ pypy/branch/dist-2.4.1/pypy/interpreter/pyparser/pythonutil.py Tue Jul 5 23:33:43 2005
@@ -50,7 +50,7 @@
pyf.close()
return pypy_parse(source, 'exec', lineno)
-def pypy_parse(source, mode='exec', lineno=False):
+def pypy_parse(source, mode='exec', lineno=False, flags=0):
"""parse <source> using PyPy's parser module and return
a tuple of three elements :
- The encoding declaration symbol or None if there were no encoding
@@ -63,7 +63,7 @@
"""
builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
target_rule = TARGET_DICT[mode]
- PYTHON_PARSER.parse_source(source, target_rule, builder)
+ PYTHON_PARSER.parse_source(source, target_rule, builder, flags)
stack_element = builder.stack[-1]
# convert the stack element into nested tuples (caution, the annotator
# can't follow this call)
More information about the Pypy-commit
mailing list