[pypy-svn] r66598 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler astcompiler/test pyparser pyparser/test

benjamin at codespeak.net benjamin at codespeak.net
Fri Jul 24 17:58:31 CEST 2009


Author: benjamin
Date: Fri Jul 24 17:58:30 2009
New Revision: 66598

Removed:
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.txt
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astgen.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/future.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/opt.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/pyassem.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/pycodegen.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symbols.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/visitor.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/astbuilder.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/asthelper.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnfgrammar.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnflexer.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnfparse.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/grammar.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonlexer.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonparse.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonutil.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/symbol.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/syntaxtree.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/support.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_lookahead.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pytokenizer.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/tuplebuilder.py
Modified:
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py
   pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
   pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py
Log:
die, old compiler, DIE!!!!


Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py	Fri Jul 24 17:58:30 2009
@@ -43,24 +43,6 @@
     return lookup
 
 
-def flatten(tup):
-    elts = []
-    for elt in tup:
-        if type(elt) == tuple:
-            elts = elts + flatten(elt)
-        else:
-            elts.append(elt)
-    return elts
-
-class Counter:
-    def __init__(self, initial):
-        self.count = initial
-
-    def next(self):
-        i = self.count
-        self.count += 1
-        return i
-
 MANGLE_LEN = 256 # magic constant from compile.c
 
 def mangle(name, klass):
@@ -87,32 +69,3 @@
             klass = klass[:end]
 
     return "_%s%s" % (klass, name)
-
-class Queue(object):
-    def __init__(self, item):
-        self.head = [item]
-        self.tail = []
-
-    def pop(self):
-        if self.head:
-            return self.head.pop()
-        else:
-            for i in range(len(self.tail)-1, -1, -1):
-                self.head.append(self.tail[i])
-            self.tail = []
-            return self.head.pop()
-
-    def extend(self, items):
-        self.tail.extend(items)
-
-    def nonempty(self):
-        return self.tail or self.head
-
-def set_filename(filename, tree):
-    """Set the filename attribute to filename on every node in tree"""
-    worklist = Queue(tree)
-    while worklist.nonempty():
-        node = worklist.pop()
-        assert isinstance(node, ast.Node)
-        node.filename = filename
-        worklist.extend(node.getChildNodes())

Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py	Fri Jul 24 17:58:30 2009
@@ -1,9 +1,6 @@
 import py
 from pypy.interpreter.astcompiler import codegen, astbuilder
 from pypy.interpreter.pyparser import pyparse
-
-from pypy.interpreter.astcompiler import misc, pycodegen, opt
-from pypy.interpreter.pyparser.test.support import source2ast
 from pypy.interpreter.pyparser.test import expressions
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.pyparser.error import SyntaxError, IndentationError

Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py	Fri Jul 24 17:58:30 2009
@@ -224,11 +224,12 @@
         self.compiler_flags = self.futureFlags.allowed_flags
 
     def compile(self, source, filename, mode, flags):
-        from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
+        from pypy.interpreter.pyparser.error import (SyntaxError,
+                                                     IndentationError,
+                                                     TokenIndentationError)
         from pypy.interpreter.pycode import PyCode
         from pypy.interpreter.pyparser.pyparse import CompileInfo
         from pypy.interpreter.pyparser.future import getFutures
-        from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError
         from pypy.interpreter.astcompiler.astbuilder import ast_from_node
         from pypy.interpreter.astcompiler.codegen import compile_ast
         from pypy.interpreter.astcompiler import consts, optimize

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py	Fri Jul 24 17:58:30 2009
@@ -1,76 +1,4 @@
-# A replacement for the token module
-#
-# adds a new map token_values to avoid doing getattr on the module
-# from PyPy RPython
-
-N_TOKENS = 0
-
-# This is used to replace None
-NULLTOKEN = -1
-
-# tok_rpunct = {}
-
-def setup_tokens(parser):
-    # global tok_rpunct
-# For compatibility, this produces the same constant values as Python 2.4.
-    parser.add_token( 'ENDMARKER' )
-    parser.add_token( 'NAME' )
-    parser.add_token( 'NUMBER' )
-    parser.add_token( 'STRING' )
-    parser.add_token( 'NEWLINE' )
-    parser.add_token( 'INDENT' )
-    parser.add_token( 'DEDENT' )
-    parser.add_token( 'LPAR',            "(" )
-    parser.add_token( 'RPAR',            ")" )
-    parser.add_token( 'LSQB',            "[" )
-    parser.add_token( 'RSQB',            "]" )
-    parser.add_token( 'COLON',           ":" )
-    parser.add_token( 'COMMA',           "," )
-    parser.add_token( 'SEMI',            ";" )
-    parser.add_token( 'PLUS',            "+" )
-    parser.add_token( 'MINUS',           "-" )
-    parser.add_token( 'STAR',            "*" )
-    parser.add_token( 'SLASH',           "/" )
-    parser.add_token( 'VBAR',            "|" )
-    parser.add_token( 'AMPER',           "&" )
-    parser.add_token( 'LESS',            "<" )
-    parser.add_token( 'GREATER',         ">" )
-    parser.add_token( 'EQUAL',           "=" )
-    parser.add_token( 'DOT',             "." )
-    parser.add_token( 'PERCENT',         "%" )
-    parser.add_token( 'BACKQUOTE',       "`" )
-    parser.add_token( 'LBRACE',          "{" )
-    parser.add_token( 'RBRACE',          "}" )
-    parser.add_token( 'EQEQUAL',         "==" )
-    ne = parser.add_token( 'NOTEQUAL',   "!=" )
-    parser.tok_values["<>"] = ne
-    parser.add_token( 'LESSEQUAL',       "<=" )
-    parser.add_token( 'GREATEREQUAL',    ">=" )
-    parser.add_token( 'TILDE',           "~" )
-    parser.add_token( 'CIRCUMFLEX',      "^" )
-    parser.add_token( 'LEFTSHIFT',       "<<" )
-    parser.add_token( 'RIGHTSHIFT',      ">>" )
-    parser.add_token( 'DOUBLESTAR',      "**" )
-    parser.add_token( 'PLUSEQUAL',       "+=" )
-    parser.add_token( 'MINEQUAL',        "-=" )
-    parser.add_token( 'STAREQUAL',       "*=" )
-    parser.add_token( 'SLASHEQUAL',      "/=" )
-    parser.add_token( 'PERCENTEQUAL',    "%=" )
-    parser.add_token( 'AMPEREQUAL',      "&=" )
-    parser.add_token( 'VBAREQUAL',       "|=" )
-    parser.add_token( 'CIRCUMFLEXEQUAL', "^=" )
-    parser.add_token( 'LEFTSHIFTEQUAL',  "<<=" )
-    parser.add_token( 'RIGHTSHIFTEQUAL', ">>=" )
-    parser.add_token( 'DOUBLESTAREQUAL', "**=" )
-    parser.add_token( 'DOUBLESLASH',     "//" )
-    parser.add_token( 'DOUBLESLASHEQUAL',"//=" )
-    parser.add_token( 'AT',              "@" )
-    parser.add_token( 'OP' )
-    parser.add_token( 'ERRORTOKEN' )
-
-# extra PyPy-specific tokens
-    parser.add_token( "COMMENT" )
-    parser.add_token( "NL" )
+"""Python token definitions."""
 
 python_tokens = {}
 python_opmap = {}

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py	Fri Jul 24 17:58:30 2009
@@ -1,50 +1,3 @@
-from pypy.interpreter.pyparser.asthelper import get_atoms
-from pypy.interpreter.pyparser.grammar import Parser
-from pypy.interpreter.pyparser import error
-
-
-def test_symbols():
-    p = Parser()
-    x1 = p.add_symbol('sym')
-    x2 = p.add_token('tok')
-    x3 = p.add_anon_symbol(':sym')
-    x4 = p.add_anon_symbol(':sym1')
-    # test basic numbering assumption
-    # symbols and tokens are attributed sequentially
-    # using the same counter
-    assert x2 == x1 + 1
-    # anon symbols have negative value
-    assert x3 != x2 + 1
-    assert x4 == x3 - 1
-    assert x3 < 0
-    y1 = p.add_symbol('sym')
-    assert y1 == x1
-    y2 = p.add_token('tok')
-    assert y2 == x2
-    y3 = p.add_symbol(':sym')
-    assert y3 == x3
-    y4 = p.add_symbol(':sym1')
-    assert y4 == x4
-
-
-def test_load():
-    d = { 5 : 'sym1',
-          6 : 'sym2',
-          9 : 'sym3',
-          }
-    p = Parser()
-    p.load_symbols( d )
-    v = p.add_symbol('sym4')
-    # check that we avoid numbering conflicts
-    assert v>9
-    v = p.add_symbol( 'sym1' )
-    assert v == 5
-    v = p.add_symbol( 'sym2' )
-    assert v == 6
-    v = p.add_symbol( 'sym3' )
-    assert v == 9
-
-
 # New parser tests.
 import py
 import tokenize



More information about the Pypy-commit mailing list