[pypy-svn] r66452 - in pypy/branch/parser-compiler: lib-python/modified-2.5.2 pypy/config pypy/module/symbol pypy/module/token pypy/module/token/test

benjamin at codespeak.net benjamin at codespeak.net
Mon Jul 20 18:56:17 CEST 2009


Author: benjamin
Date: Mon Jul 20 18:56:13 2009
New Revision: 66452

Added:
   pypy/branch/parser-compiler/pypy/module/token/
   pypy/branch/parser-compiler/pypy/module/token/__init__.py   (contents, props changed)
   pypy/branch/parser-compiler/pypy/module/token/test/
   pypy/branch/parser-compiler/pypy/module/token/test/test_token.py   (contents, props changed)
Removed:
   pypy/branch/parser-compiler/lib-python/modified-2.5.2/symbol.py
   pypy/branch/parser-compiler/lib-python/modified-2.5.2/token.py
Modified:
   pypy/branch/parser-compiler/pypy/config/pypyoption.py
   pypy/branch/parser-compiler/pypy/module/symbol/__init__.py
Log:
implement token and symbol as builtin modules

Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/config/pypyoption.py	(original)
+++ pypy/branch/parser-compiler/pypy/config/pypyoption.py	Mon Jul 20 18:56:13 2009
@@ -18,7 +18,7 @@
 default_modules.update(dict.fromkeys(
     ["_codecs", "gc", "_weakref", "marshal", "errno",
      "math", "_sre", "_pickle_support", "operator",
-     "parser", "symbol", "_random", "__pypy__"]))
+     "parser", "symbol", "token", "_random", "__pypy__"]))
 
 
 # --allworkingmodules

Modified: pypy/branch/parser-compiler/pypy/module/symbol/__init__.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/module/symbol/__init__.py	(original)
+++ pypy/branch/parser-compiler/pypy/module/symbol/__init__.py	Mon Jul 20 18:56:13 2009
@@ -5,10 +5,7 @@
 """
 
 from pypy.interpreter.mixedmodule import MixedModule
-
-# Forward imports so they run at startup time
-import pypy.interpreter.pyparser.pythonlexer
-import pypy.interpreter.pyparser.pythonparse
+from pypy.interpreter.pyparser import pygram
 
 
 class Module(MixedModule):
@@ -17,23 +14,11 @@
     interpleveldefs = {}     # see below
 
 
-# Export the values from our custom symbol module.
-# Skip negative values (the corresponding symbols are not visible in
-# pure Python).
-sym_name = {}
-
-def _init_symbols(grammar_version):
-    global sym_name
-
+def _init_symbols():
     sym_name = {}
-    from pypy.interpreter.pyparser.pythonparse import make_pyparser
-    parser = make_pyparser(grammar_version)
-
-    for name, val in parser.symbols.items():
-        if val >= 0:
-            Module.interpleveldefs[name] = 'space.wrap(%d)' % val
-            sym_name[val] = name
+    for name, val in pygram.python_grammar.symbol_ids.iteritems():
+        Module.interpleveldefs[name] = 'space.wrap(%d)' % val
+        sym_name[val] = name
     Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
 
-# This is very evil
-_init_symbols('2.5')
+_init_symbols()

Added: pypy/branch/parser-compiler/pypy/module/token/__init__.py
==============================================================================
--- (empty file)
+++ pypy/branch/parser-compiler/pypy/module/token/__init__.py	Mon Jul 20 18:56:13 2009
@@ -0,0 +1,38 @@
+from pypy.interpreter.mixedmodule import MixedModule
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.pyparser import pytoken, pygram
+
+
+class Module(MixedModule):
+
+    appleveldefs = {}
+    interpleveldefs = {
+        "NT_OFFSET" : "space.wrap(256)",
+        "ISTERMINAL" : "__init__.isterminal",
+        "ISNONTERMINAL" : "__init__.isnonterminal",
+        "ISEOF" : "__init__.iseof"
+        }
+
+
+def _init_tokens():
+    tok_name = {}
+    for tok, id in pytoken.python_tokens.iteritems():
+        Module.interpleveldefs[tok] = "space.wrap(%d)" % (id,)
+        tok_name[id] = tok
+    Module.interpleveldefs["tok_name"] = "space.wrap(%r)" % (tok_name,)
+    Module.interpleveldefs["N_TOKENS"] = "space.wrap(%d)" % len(tok_name)
+
+_init_tokens()
+
+
+def isterminal(space, tok):
+    return space.wrap(tok < 256)
+isterminal.unwrap_spec = [ObjSpace, int]
+
+def isnonterminal(space, tok):
+    return space.wrap(tok >= 256)
+isnonterminal.unwrap_spec = [ObjSpace, int]
+
+def iseof(space, tok):
+    return space.wrap(tok == pygram.tokens.ENDMARKER)
+iseof.unwrap_spec = [ObjSpace, int]

Added: pypy/branch/parser-compiler/pypy/module/token/test/test_token.py
==============================================================================
--- (empty file)
+++ pypy/branch/parser-compiler/pypy/module/token/test/test_token.py	Mon Jul 20 18:56:13 2009
@@ -0,0 +1,18 @@
+class AppTestToken:
+
+    def setup_class(cls):
+        cls.w_token = cls.space.appexec([], """():
+    import token
+    return token""")
+
+    def test_isterminal(self):
+        assert self.token.ISTERMINAL(self.token.ENDMARKER)
+        assert not self.token.ISTERMINAL(300)
+
+    def test_isnonterminal(self):
+        assert self.token.ISNONTERMINAL(300)
+        assert not self.token.ISNONTERMINAL(self.token.NAME)
+
+    def test_iseof(self):
+        assert self.token.ISEOF(self.token.ENDMARKER)
+        assert not self.token.ISEOF(self.token.NAME)



More information about the Pypy-commit mailing list