[pypy-svn] r14389 - pypy/dist/pypy/interpreter/pyparser

[arigo at codespeak.net]

Author: arigo
Date: Thu Jul  7 17:34:07 2005
New Revision: 14389

Modified:
   pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
   pypy/dist/pypy/interpreter/pyparser/pytokenize.py
   pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
Log:
Hack to support the '@' token on top of Python 2.3.

Some notes about the pyparser:  I realize that it was a quick hack, but we
should keep in mind that at some point we need something more clearly
independent of CPython's interp-level standard library 'symbol' and 'token'
modules.  I suppose that it should also be refactored to isolate in non-global
instances the tokens and symbols to use, in the same way as the grammar is
isolated.



Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonlexer.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py	Thu Jul  7 17:34:07 2005
@@ -2,7 +2,7 @@
 it obeys the TokenSource interface defined for the grammar
 analyser in grammar.py
 """
-import symbol, sys
+import sys
 from codeop import PyCF_DONT_IMPLY_DEDENT
 
 from pypy.interpreter.pyparser.grammar import TokenSource, Token

Modified: pypy/dist/pypy/interpreter/pyparser/pytokenize.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pytokenize.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pytokenize.py	Thu Jul  7 17:34:07 2005
@@ -52,7 +52,8 @@
      '/': 11, '0': 4, '1': 5, '2': 5,
      '3': 5, '4': 5, '5': 5, '6': 5,
      '7': 5, '8': 5, '9': 5, ':': 13,
-     ';': 13, '<': 9, '=': 12, '>': 8, 'A': 1,
+     ';': 13, '<': 9, '=': 12, '>': 8,
+     '@': 13, 'A': 1,
      'B': 1, 'C': 1, 'D': 1, 'E': 1,
      'F': 1, 'G': 1, 'H': 1, 'I': 1,
      'J': 1, 'K': 1, 'L': 1, 'M': 1,

Modified: pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/syntaxtree.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/syntaxtree.py	Thu Jul  7 17:34:07 2005
@@ -2,6 +2,11 @@
 import symbol
 import token
 
+# XXX hack: make sure '@' is in the token list
+if not hasattr(token, 'AT'):
+    token.AT = token.N_TOKENS + 2    # see pythonlexer.py for why '+2'
+    token.tok_name[token.AT] = 'AT'
+
 TOKEN_MAP = {
     "STRING" : token.STRING,
     "NUMBER" : token.NUMBER,
@@ -60,6 +65,7 @@
     "~" : token.TILDE,
     "|" : token.VBAR,
     "|=" : token.VBAREQUAL,
+    "@": token.AT,
     }
 NT_OFFSET = token.NT_OFFSET