[pypy-svn] r77617 - in pypy/branch/fast-forward/pypy/interpreter: astcompiler astcompiler/test pyparser pyparser/test

afa at codespeak.net afa at codespeak.net
Tue Oct 5 18:53:43 CEST 2010


Author: afa
Date: Tue Oct  5 18:53:41 2010
New Revision: 77617

Modified:
   pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py
   pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
Add new octal literal syntax 0o777 to the tokenizer.


Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py	Tue Oct  5 18:53:41 2010
@@ -1037,13 +1037,15 @@
         if raw.startswith("0"):
             if len(raw) > 2 and raw[1] in "Xx":
                 base = 16
+            ## elif len(raw) > 2 and raw[1] in "Oo": # Fallback below is enough
+            ##     base = 8
             elif len(raw) > 1:
                 base = 8
             # strip leading characters
             i = 0
             limit = len(raw) - 1
             while i < limit:
-                if raw[i] not in "0xX":
+                if raw[i] not in "0xXoO":
                     break
                 i += 1
             raw = raw[i:]

Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py	Tue Oct  5 18:53:41 2010
@@ -1137,6 +1137,8 @@
         assert space.eq_w(get_num("-0xAAAAAAL"), space.wrap(-0xAAAAAAL))
         n = get_num(str(-sys.maxint - 1))
         assert space.is_true(space.isinstance(n, space.w_int))
+        for num in ("0o53", "0O53", "0o0000053", "0O00053"):
+            assert space.eq_w(get_num(num), space.wrap(053))
 
     def check_comprehension(self, brackets, ast_type):
         def brack(s):

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py	Tue Oct  5 18:53:41 2010
@@ -58,6 +58,7 @@
                       maybe(states, groupStr(states, "lL")))
     octNumber = chain(states,
                       newArcPair(states, "0"),
+                      maybe(states, groupStr(states, "oO")),
                       any(states, groupStr(states, "01234567")),
                       maybe(states, groupStr(states, "lL")))
     decNumber = chain(states,

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py	Tue Oct  5 18:53:41 2010
@@ -26,9 +26,9 @@
 accepts = [True, True, True, True, True, True, True, True,
            True, True, False, True, True, True, False, False,
            False, False, True, False, False, True, True,
-           False, True, False, True, False, True, False,
-           True, False, False, False, True, False, False,
-           False, True]
+           True, False, True, False, True, False, True,
+           False, True, False, False, False, True, False,
+           False, False, True]
 states = [
     {'\t': 0, '\n': 13, '\x0c': 0,
      '\r': 14, ' ': 0, '!': 10, '"': 16,
@@ -109,21 +109,22 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1},
 
-    {'.': 24, '0': 22, '1': 22, '2': 22,
-     '3': 22, '4': 22, '5': 22, '6': 22,
-     '7': 22, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'L': 13, 'X': 21, 'e': 25,
-     'j': 13, 'l': 13, 'x': 21},
+    {'.': 25, '0': 23, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 24, '9': 24, 'E': 26,
+     'J': 13, 'L': 13, 'O': 22, 'X': 21,
+     'e': 26, 'j': 13, 'l': 13, 'o': 22,
+     'x': 21},
 
-    {'.': 24, '0': 5, '1': 5, '2': 5,
+    {'.': 25, '0': 5, '1': 5, '2': 5,
      '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+     '7': 5, '8': 5, '9': 5, 'E': 26,
+     'J': 13, 'L': 13, 'e': 26, 'j': 13,
      'l': 13},
 
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26},
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27},
 
     {'*': 12, '=': 13},
 
@@ -141,92 +142,101 @@
 
     {'\n': 13},
 
-    {automata.DEFAULT: 19, '\n': 27,
-     "'": 28, '\\': 29},
+    {automata.DEFAULT: 19, '\n': 28,
+     "'": 29, '\\': 30},
 
-    {automata.DEFAULT: 20, '\n': 27,
-     '"': 30, '\\': 31},
+    {automata.DEFAULT: 20, '\n': 28,
+     '"': 31, '\\': 32},
 
     {'\n': 13, '\r': 14},
 
-    {automata.DEFAULT: 18, '\n': 27,
-     '\r': 27},
+    {automata.DEFAULT: 18, '\n': 28,
+     '\r': 28},
 
-    {automata.DEFAULT: 19, '\n': 27,
-     "'": 13, '\\': 29},
+    {automata.DEFAULT: 19, '\n': 28,
+     "'": 13, '\\': 30},
 
-    {automata.DEFAULT: 20, '\n': 27,
-     '"': 13, '\\': 31},
+    {automata.DEFAULT: 20, '\n': 28,
+     '"': 13, '\\': 32},
 
     {'0': 21, '1': 21, '2': 21, '3': 21,
      '4': 21, '5': 21, '6': 21, '7': 21,
      '8': 21, '9': 21, 'A': 21, 'B': 21,
      'C': 21, 'D': 21, 'E': 21, 'F': 21,
      'L': 13, 'a': 21, 'b': 21, 'c': 21,
-     'd': 21, 'e': 21, 'f': 21, 'l': 13},
-
-    {'.': 24, '0': 22, '1': 22, '2': 22,
-     '3': 22, '4': 22, '5': 22, '6': 22,
-     '7': 22, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+     'd': 21, 'e': 21, 'f': 21,
      'l': 13},
 
-    {'.': 24, '0': 23, '1': 23, '2': 23,
+    {'0': 22, '1': 22, '2': 22, '3': 22,
+     '4': 22, '5': 22, '6': 22, '7': 22,
+     'L': 13, 'l': 13},
+
+    {'.': 25, '0': 23, '1': 23, '2': 23,
      '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
+     '7': 23, '8': 24, '9': 24, 'E': 26,
+     'J': 13, 'L': 13, 'e': 26, 'j': 13,
+     'l': 13},
 
-    {'0': 24, '1': 24, '2': 24, '3': 24,
-     '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24, 'E': 32, 'J': 13,
-     'e': 32, 'j': 13},
-
-    {'+': 33, '-': 33, '0': 34, '1': 34,
-     '2': 34, '3': 34, '4': 34, '5': 34,
-     '6': 34, '7': 34, '8': 34, '9': 34},
-
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 32, 'J': 13,
-     'e': 32, 'j': 13},
+    {'.': 25, '0': 24, '1': 24, '2': 24,
+     '3': 24, '4': 24, '5': 24, '6': 24,
+     '7': 24, '8': 24, '9': 24, 'E': 26,
+     'J': 13, 'e': 26, 'j': 13},
+
+    {'0': 25, '1': 25, '2': 25, '3': 25,
+     '4': 25, '5': 25, '6': 25, '7': 25,
+     '8': 25, '9': 25, 'E': 33, 'J': 13,
+     'e': 33, 'j': 13},
+
+    {'+': 34, '-': 34, '0': 35, '1': 35,
+     '2': 35, '3': 35, '4': 35, '5': 35,
+     '6': 35, '7': 35, '8': 35,
+     '9': 35},
+
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27, 'E': 33, 'J': 13,
+     'e': 33, 'j': 13},
 
     {},
 
     {"'": 13},
 
-    {automata.DEFAULT: 35, '\n': 13,
+    {automata.DEFAULT: 36, '\n': 13,
      '\r': 14},
 
     {'"': 13},
 
-    {automata.DEFAULT: 36, '\n': 13,
+    {automata.DEFAULT: 37, '\n': 13,
      '\r': 14},
 
-    {'+': 37, '-': 37, '0': 38, '1': 38,
-     '2': 38, '3': 38, '4': 38, '5': 38,
-     '6': 38, '7': 38, '8': 38, '9': 38},
-
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34},
-
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'J': 13, 'j': 13},
-
-    {automata.DEFAULT: 35, '\n': 27,
-     "'": 13, '\\': 29},
-
-    {automata.DEFAULT: 36, '\n': 27,
-     '"': 13, '\\': 31},
-
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38},
-
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38, 'J': 13, 'j': 13},
+    {'+': 38, '-': 38, '0': 39, '1': 39,
+     '2': 39, '3': 39, '4': 39, '5': 39,
+     '6': 39, '7': 39, '8': 39,
+     '9': 39},
+
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35,
+     '8': 35, '9': 35},
+
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35,
+     '8': 35, '9': 35, 'J': 13,
+     'j': 13},
+
+    {automata.DEFAULT: 36, '\n': 28,
+     "'": 13, '\\': 30},
+
+    {automata.DEFAULT: 37, '\n': 28,
+     '"': 13, '\\': 32},
+
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39},
+
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'J': 13,
+     'j': 13},
 
     ]
 pseudoDFA = automata.DFA(states, accepts)

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py	Tue Oct  5 18:53:41 2010
@@ -108,3 +108,8 @@
         self.parse("br'\\\n'")
 
         py.test.raises(SyntaxError, self.parse, "b'a\\n")
+
+    def test_new_octal_literal(self):
+        self.parse('0777')
+        self.parse('0o777')
+        py.test.raises(SyntaxError, self.parse, "0o778")



More information about the Pypy-commit mailing list