[pypy-svn] r48908 - in pypy/branch/dist-future-fixing/pypy/interpreter: . pyparser pyparser/test

ac at codespeak.net ac at codespeak.net
Wed Nov 21 17:05:03 CET 2007


Author: ac
Date: Wed Nov 21 17:05:03 2007
New Revision: 48908

Modified:
   pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py
   pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py
Log:
(jacob, arre)
__future__ imports and 'with'-statements work again. Yay!



Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pycompiler.py	Wed Nov 21 17:05:03 2007
@@ -2,6 +2,8 @@
 General classes for bytecode compilers.
 Compiler instances are stored into 'space.getexecutioncontext().compiler'.
 """
+
+import sys
 from codeop import PyCF_DONT_IMPLY_DEDENT
 from pypy.interpreter.error import OperationError
 
@@ -111,7 +113,7 @@
         else:
             return 0
 
-from pypy.interpreter.pyparser.future import futureFlags
+from pypy.interpreter.pyparser import future
 
 class CPythonCompiler(PyCodeCompiler):
     """Faked implementation of a compiler, using the underlying compile()."""
@@ -119,8 +121,10 @@
     def __init__(self, space):
         self.space = space
         self.w_compile_hook = space.w_None
-
-        self.compiler_flags = futureFlags.allowed_flags
+        if sys.version_info >= (2.5):
+            self.compiler_flags = future.futureFlags_2_5.allowed_flags
+        else:
+            self.compiler_flags = future.futureFlags_2_4.allowed_flags
 
     def compile(self, source, filename, mode, flags):
         from pypy.tool import stdlib___future__
@@ -218,8 +222,11 @@
         self.grammar_version = override_version or space.config.objspace.pyversion
         self.parser = make_pyparser(self.grammar_version)
         self.additional_rules = {}
-
-        self.compiler_flags = futureFlags.allowed_flags
+        if self.grammar_version >= '2.5':
+            self.futureFlags = future.futureFlags_2_5
+        else:
+            self.futureFlags = future.futureFlags_2_4
+        self.compiler_flags = self.futureFlags.allowed_flags
 
     def compile(self, source, filename, mode, flags):
         from pyparser.error import SyntaxError
@@ -241,7 +248,7 @@
             for rulename, buildfunc in self.additional_rules.iteritems():
                 assert isinstance(buildfunc, Function)
                 builder.user_build_rules[rulename] = buildfunc
-            flags |= getFutures(source)
+            flags |= getFutures(self.futureFlags, source)
             self.parser.parse_source(source, mode, builder, flags)
             ast_tree = builder.rule_stack[-1]
             encoding = builder.source_encoding
@@ -261,7 +268,7 @@
                 raise
         try:
             astcompiler.misc.set_filename(filename, ast_tree)
-            flag_names = futureFlags.get_flag_names(space, flags)
+            flag_names = self.futureFlags.get_flag_names(space, flags)
             if mode == 'exec':
                 codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names)
             elif mode == 'single':

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/astbuilder.py	Wed Nov 21 17:05:03 2007
@@ -869,13 +869,6 @@
             names.append((name, as_name))
             if index < l: # case ','
                 index += 1
-##     if from_name == '__future__':
-##         for name, asname in names:
-##             if name == 'with_statement':
-##                 # found from __future__ import with_statement
-##                 if not builder.with_enabled:
-##                     builder.enable_with()
-##                     #raise pythonparse.AlternateGrammarException()
     builder.push(ast.From(from_name, names, atoms[0].lineno))
 
 
@@ -1069,17 +1062,9 @@
         self.rule_stack = []
         self.space = space
         self.source_encoding = None
-##        self.with_enabled = False
         self.build_rules = ASTRULES_Template
         self.user_build_rules = {}
 
-##     def enable_with(self):
-##         if self.with_enabled:
-##             return
-##         self.with_enabled = True
-##         # XXX
-##         # self.keywords.update({'with':None, 'as': None})
-
     def context(self):
         return AstBuilderContext(self.rule_stack)
 

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/ebnfparse.py	Wed Nov 21 17:05:03 2007
@@ -40,9 +40,6 @@
 
 class NameToken(Token):
     """A token that is not a keyword"""
-    def __init__(self, parser, keywords=None):
-        Token.__init__(self, parser.tokens['NAME'])
-        self.keywords = keywords
 
     def match(self, source, builder, level=0):
         """Matches a token.
@@ -60,7 +57,7 @@
         if tk.codename == self.codename:
             # XXX (adim): this is trunk's keyword management
             # if tk.value not in builder.keywords:
-            if tk.value not in self.keywords:
+            if not tk.isKeyword:
                 ret = builder.token( tk.codename, tk.value, source )
                 return ret
         source.restore( ctx )
@@ -78,7 +75,7 @@
             return False
         # XXX (adim): this is trunk's keyword management
         # if other.value in builder.keywords:
-        if other.value in self.keywords:
+        if other.isKeyword:
             return False
         return True
 
@@ -107,7 +104,7 @@
         self.keywords = []
         NAME = dest_parser.add_token(Token('NAME'))
         # NAME = dest_parser.tokens['NAME']
-        self.tokens[NAME] = NameToken(dest_parser, keywords=self.keywords)
+        self.tokens[NAME] = NameToken(NAME)
 
         # XXX Temporary. We should be able to get rid of it later
         self.parser = dest_parser

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/future.py	Wed Nov 21 17:05:03 2007
@@ -27,8 +27,8 @@
 from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \
     CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT
             
-def getFutures(source):
-    futures = FutureAutomaton(source)
+def getFutures(futureFlags, source):
+    futures = FutureAutomaton(futureFlags, source)
     try:
         futures.start()
     except (IndexError, DoneException), e:
@@ -62,7 +62,8 @@
     precede a future statement.
     """
     
-    def __init__(self, string):
+    def __init__(self, futureFlags, string):
+        self.futureFlags = futureFlags
         self.s = string
         self.pos = 0
         self.docstringConsumed = False
@@ -237,7 +238,7 @@
 
     def setFlag(self, feature):
         try:
-            self.flags |= futureFlags.compiler_features[feature]
+            self.flags |= self.futureFlags.compiler_features[feature]
         except IndexError:
             pass
 
@@ -271,12 +272,5 @@
                 flag_names.append(name)
         return flag_names
 
-# XXX This is a hack to deal with the fact that we currently are
-#     using the Python 2.4.1 libraries even when running Python 2.5
-#     and that we have a hacked __future__ module.
-from pypy.config.pypyoption import get_pypy_config
-config = get_pypy_config(translating=False)
-if config.objspace.pyversion == '2.4' and False:
-    futureFlags = FutureFlags((2, 4, 4, 'final', 0))
-else:
-    futureFlags = FutureFlags((2, 5, 0, 'final', 0))
+futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0))
+futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0))

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/grammar.py	Wed Nov 21 17:05:03 2007
@@ -655,6 +655,7 @@
 
 class Token(GrammarElement):
     """Represents a Token in a grammar rule (a lexer token)"""
+    isKeyword = True
     def __init__(self, codename, value=None):
         GrammarElement.__init__(self, codename)
         self.value = value
@@ -679,7 +680,7 @@
 
         ctx = source.context()
         tk = source.next()
-        if tk.codename == self.codename:
+        if tk.codename == self.codename and tk.isKeyword:
             if self.value is None:
                 ret = builder.token( tk.codename, tk.value, source )
                 return ret
@@ -713,7 +714,7 @@
         # if (self.value is not None and builder.keywords is not None
         #     and self.value not in builder.keywords):
         #     return False
-        res = other.codename == self.codename and self.value in [None, other.value]
+        res = other.isKeyword and other.codename == self.codename and self.value in [None, other.value]
         #print "matching", self, other, res
         return res
 

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonlexer.py	Wed Nov 21 17:05:03 2007
@@ -62,7 +62,7 @@
         SyntaxError.__init__(self, msg, lineno, offset, line)
         self.token_stack = token_stack
 
-def generate_tokens(parser, lines, flags):
+def generate_tokens(parser, lines, flags, keywords):
     """
     This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
     the original function is not RPYTHON (uses yield)
@@ -251,6 +251,8 @@
                         last_comment = ''
                 elif initial in namechars:                 # ordinary name
                     tok = Token(parser.tokens['NAME'], token)
+                    if token not in keywords:
+                        tok.isKeyword = False
                     token_list.append((tok, line, lnum, pos))
                     last_comment = ''
                 elif initial == '\\':                      # continued stmt
@@ -312,12 +314,9 @@
 
 class PythonSource(TokenSource):
     """This source uses Jonathan's tokenizer"""
-    def __init__(self, parser, strings, flags=0):
-        # TokenSource.__init__(self)
-        #self.parser = parser
-        
+    def __init__(self, parser, strings, keywords, flags=0):
         self.input = strings
-        tokens = generate_tokens(parser, strings, flags)
+        tokens = generate_tokens(parser, strings, flags, keywords)
         self.token_stack = tokens
         self._current_line = '' # the current line (as a string)
         self._lineno = -1
@@ -393,14 +392,3 @@
 
 Source = PythonSource
 
-def tokenize_file(filename):
-    f = file(filename).read()
-    src = Source(f)
-    token = src.next()
-    while token != ("ENDMARKER", None) and token != (None, None):
-        print token
-        token = src.next()
-
-if __name__ == '__main__':
-    import sys
-    tokenize_file(sys.argv[1])

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/pythonparse.py	Wed Nov 21 17:05:03 2007
@@ -10,7 +10,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser.pythonlexer import Source, match_encoding_declaration
-#from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT
+from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT
 # XXX seems dead
 #import pypy.interpreter.pyparser.pysymbol as pysymbol
 import pypy.interpreter.pyparser.pytoken as pytoken
@@ -124,12 +124,20 @@
 
 
     def parse_lines(self, lines, goal, builder, flags=0):
-        # builder.keywords = self.keywords.copy()
-        # if flags & CO_FUTURE_WITH_STATEMENT:
-        #     builder.enable_with()
         goalnumber = self.symbols[goal]
         target = self.root_rules[goalnumber]
-        src = Source(self, lines, flags)
+        keywords = {} # dict.fromkeys(self.keywords)
+        for keyword in self.keywords:
+            keywords[keyword] = None
+       
+        if flags & CO_FUTURE_WITH_STATEMENT:
+	    keywords.update({'with': None,
+                             'as': None })
+        else:
+            keywords.pop('with', None)
+            keywords.pop('as', None)
+        src = Source(self, lines, keywords, flags)
+
         if not target.match(src, builder):
             line, lineno = src.debug()
             # XXX needs better error messages

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_futureautomaton.py	Wed Nov 21 17:05:03 2007
@@ -3,7 +3,7 @@
 from pypy.tool import stdlib___future__ as fut
 
 def run(s):
-    f = future.FutureAutomaton(s)
+    f = future.FutureAutomaton(future.futureFlags_2_5, s)
     try:
         f.start()
     except IndexError, future.DoneException:
@@ -124,12 +124,12 @@
 
 def test_full_chain():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,);  from __future__ import with_statement\n'
-    flags = future.getFutures(s)
+    flags = future.getFutures(future.futureFlags_2_5, s)
     assert flags == (fut.CO_FUTURE_DIVISION |
                      fut.CO_GENERATOR_ALLOWED |
                      fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_intervening_code():
     s = 'from  __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n'
-    flags = future.getFutures(s)
+    flags = future.getFutures(future.futureFlags_2_5, s)
     assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0

Modified: pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py
==============================================================================
--- pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py	(original)
+++ pypy/branch/dist-future-fixing/pypy/interpreter/pyparser/test/test_pytokenizer.py	Wed Nov 21 17:05:03 2007
@@ -18,7 +18,7 @@
 
 def parse_source(source):
     """returns list of parsed tokens"""
-    lexer = Source( P, source.splitlines(True))
+    lexer = Source( P, source.splitlines(True), {})
     tokens = []
     last_token = Token(NULLTOKEN, None)
     while last_token.codename != ENDMARKER:



More information about the Pypy-commit mailing list