[pypy-svn] r15230 - in pypy/dist/pypy: interpreter interpreter/pyparser module/symbol

[arigo at codespeak.net]

Author: arigo
Date: Thu Jul 28 12:25:31 2005
New Revision: 15230

Added:
   pypy/dist/pypy/module/symbol/   (props changed)
   pypy/dist/pypy/module/symbol/__init__.py   (contents, props changed)
Modified:
   pypy/dist/pypy/interpreter/baseobjspace.py
   pypy/dist/pypy/interpreter/pyparser/pytoken.py
Log:
An attempt to bring in sync the interp-level and app-level numbers associated
to the tokens and symbols:

* The token list is now hand-coded in pytoken.py, in such an order that it
  produces the same numbers as the lib-python/2.4.1/token.py.

* The symbol list cannot be hand-coded, as it is generated by the grammar
  parser.  Instead, there is now a mixed module 'symbol' that exports the
  values produced by the grammar parser to app-level.  This allows
  grammar experimentations without having to worry about the values getting
  out of sync.



Modified: pypy/dist/pypy/interpreter/baseobjspace.py
==============================================================================
--- pypy/dist/pypy/interpreter/baseobjspace.py	(original)
+++ pypy/dist/pypy/interpreter/baseobjspace.py	Thu Jul 28 12:25:31 2005
@@ -147,6 +147,7 @@
             builtinmodule_list.append(('_codecs', None))
             if self.options.useparsermodule == "recparser":
                 builtinmodule_list.append(('parser', 'recparser'))
+                builtinmodule_list.append(('symbol', None))
             elif self.options.useparsermodule == "parser":
                 builtinmodule_list.append(('parser', None))
             if self.options.nofakedmodules:

Modified: pypy/dist/pypy/interpreter/pyparser/pytoken.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pytoken.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pytoken.py	Thu Jul 28 12:25:31 2005
@@ -3,9 +3,7 @@
 # adds a new map token_values to avoid doing getattr on the module
 # from PyPy RPython
 
-import token
-
-N_TOKENS = token.N_TOKENS
+N_TOKENS = 0
 
 tok_name = {}
 tok_values = {}
@@ -23,13 +21,62 @@
 # This is used to replace None
 add_token( 'NULLTOKEN', -1 )
 
-for value, name in token.tok_name.items():
-    add_token( name, value )
-
-# Make sure '@' is in the token list
-if "AT" not in tok_values:
-    add_token( "AT" )
+# For compatibility, this produces the same constant values as Python 2.4.
+add_token( 'ENDMARKER' )
+add_token( 'NAME' )
+add_token( 'NUMBER' )
+add_token( 'STRING' )
+add_token( 'NEWLINE' )
+add_token( 'INDENT' )
+add_token( 'DEDENT' )
+add_token( 'LPAR' )
+add_token( 'RPAR' )
+add_token( 'LSQB' )
+add_token( 'RSQB' )
+add_token( 'COLON' )
+add_token( 'COMMA' )
+add_token( 'SEMI' )
+add_token( 'PLUS' )
+add_token( 'MINUS' )
+add_token( 'STAR' )
+add_token( 'SLASH' )
+add_token( 'VBAR' )
+add_token( 'AMPER' )
+add_token( 'LESS' )
+add_token( 'GREATER' )
+add_token( 'EQUAL' )
+add_token( 'DOT' )
+add_token( 'PERCENT' )
+add_token( 'BACKQUOTE' )
+add_token( 'LBRACE' )
+add_token( 'RBRACE' )
+add_token( 'EQEQUAL' )
+add_token( 'NOTEQUAL' )
+add_token( 'LESSEQUAL' )
+add_token( 'GREATEREQUAL' )
+add_token( 'TILDE' )
+add_token( 'CIRCUMFLEX' )
+add_token( 'LEFTSHIFT' )
+add_token( 'RIGHTSHIFT' )
+add_token( 'DOUBLESTAR' )
+add_token( 'PLUSEQUAL' )
+add_token( 'MINEQUAL' )
+add_token( 'STAREQUAL' )
+add_token( 'SLASHEQUAL' )
+add_token( 'PERCENTEQUAL' )
+add_token( 'AMPEREQUAL' )
+add_token( 'VBAREQUAL' )
+add_token( 'CIRCUMFLEXEQUAL' )
+add_token( 'LEFTSHIFTEQUAL' )
+add_token( 'RIGHTSHIFTEQUAL' )
+add_token( 'DOUBLESTAREQUAL' )
+add_token( 'DOUBLESLASH' )
+add_token( 'DOUBLESLASHEQUAL' )
+add_token( 'AT' )
+add_token( 'OP' )
+add_token( 'ERRORTOKEN' )
 
+# extra PyPy-specific tokens
 add_token( "COMMENT" )
 add_token( "NL" )
 

Added: pypy/dist/pypy/module/symbol/__init__.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/module/symbol/__init__.py	Thu Jul 28 12:25:31 2005
@@ -0,0 +1,31 @@
+"""Dynamic replacement for the stdlib 'symbol' module.
+
+This module exports the symbol values computed by the grammar parser
+at run-time.
+"""
+
+from pypy.interpreter.mixedmodule import MixedModule 
+
+# Forward imports so they run at startup time
+import pypy.interpreter.pyparser.pythonlexer
+import pypy.interpreter.pyparser.pythonparse
+
+
+class Module(MixedModule):
+     """Non-terminal symbols of Python grammar."""
+
+     appleveldefs = {}
+     interpleveldefs = {}     # see below
+
+
+# Export the values from our custom symbol module.
+# Skip negative values (the corresponding symbols are not visible in
+# pure Python).
+from pypy.interpreter.pyparser import pysymbol
+
+sym_name = {}
+for val, name in pysymbol.sym_name.items():
+    if val >= 0:
+        Module.interpleveldefs[name] = 'space.wrap(%d)' % val
+        sym_name[val] = name
+Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)