[pypy-commit] pypy py3k: (antocuni, romain): update the tokenizer to accept 0000* as a literal
antocuni
noreply at buildbot.pypy.org
Thu Jan 19 12:55:36 CET 2012
Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r51484:ac227cc49124
Date: 2012-01-19 12:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ac227cc49124/
Log: (antocuni, romain): update the tokenizer to accept 0000* as a
literal
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/genpytokenize.py
@@ -73,7 +73,9 @@
decNumber = chain(states,
groupStr(states, "123456789"),
any(states, makeDigits()))
- zero = newArcPair(states, "0")
+ zero = chain(states,
+ newArcPair(states, "0"),
+ any(states, newArcPair(states, "0")))
intNumber = group(states, hexNumber, octNumber, binNumber, decNumber, zero)
# ____________________________________________________________
# Exponents
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -25,10 +25,10 @@
accepts = [True, True, True, True, True, True, True, True,
True, True, False, True, True, True, True, False,
- False, False, True, False, False, False, False,
- True, False, True, False, True, False, False,
- True, False, False, True, True, True, False,
- False, True, False, False, False, True]
+ False, False, True, False, False, False, True,
+ False, True, False, True, False, True, False,
+ False, True, False, False, True, True, True,
+ False, False, True, False, False, False, True]
states = [
# 0
{'\t': 0, '\n': 13, '\x0c': 0,
@@ -110,21 +110,21 @@
'v': 1, 'w': 1, 'x': 1, 'y': 1,
'z': 1},
# 4
- {'.': 23, '0': 22, '1': 22, '2': 22,
- '3': 22, '4': 22, '5': 22, '6': 22,
- '7': 22, '8': 22, '9': 22, 'B': 21,
- 'E': 24, 'J': 13, 'O': 20, 'X': 19,
- 'b': 21, 'e': 24, 'j': 13, 'o': 20,
+ {'.': 24, '0': 22, '1': 23, '2': 23,
+ '3': 23, '4': 23, '5': 23, '6': 23,
+ '7': 23, '8': 23, '9': 23, 'B': 21,
+ 'E': 25, 'J': 13, 'O': 20, 'X': 19,
+ 'b': 21, 'e': 25, 'j': 13, 'o': 20,
'x': 19},
# 5
- {'.': 23, '0': 5, '1': 5, '2': 5,
+ {'.': 24, '0': 5, '1': 5, '2': 5,
'3': 5, '4': 5, '5': 5, '6': 5,
- '7': 5, '8': 5, '9': 5, 'E': 24,
- 'J': 13, 'e': 24, 'j': 13},
+ '7': 5, '8': 5, '9': 5, 'E': 25,
+ 'J': 13, 'e': 25, 'j': 13},
# 6
- {'0': 25, '1': 25, '2': 25, '3': 25,
- '4': 25, '5': 25, '6': 25, '7': 25,
- '8': 25, '9': 25},
+ {'0': 26, '1': 26, '2': 26, '3': 26,
+ '4': 26, '5': 26, '6': 26, '7': 26,
+ '8': 26, '9': 26},
# 7
{'*': 12, '=': 13},
# 8
@@ -142,100 +142,105 @@
# 14
{'\n': 13},
# 15
- {automata.DEFAULT: 29, '\n': 26,
- '\r': 26, "'": 27, '\\': 28},
+ {automata.DEFAULT: 30, '\n': 27,
+ '\r': 27, "'": 28, '\\': 29},
# 16
- {automata.DEFAULT: 32, '\n': 26,
- '\r': 26, '"': 30, '\\': 31},
+ {automata.DEFAULT: 33, '\n': 27,
+ '\r': 27, '"': 31, '\\': 32},
# 17
{'\n': 13, '\r': 14},
# 18
- {automata.DEFAULT: 18, '\n': 26, '\r': 26},
+ {automata.DEFAULT: 18, '\n': 27, '\r': 27},
# 19
- {'0': 33, '1': 33, '2': 33, '3': 33,
- '4': 33, '5': 33, '6': 33, '7': 33,
- '8': 33, '9': 33, 'A': 33, 'B': 33,
- 'C': 33, 'D': 33, 'E': 33, 'F': 33,
- 'a': 33, 'b': 33, 'c': 33, 'd': 33,
- 'e': 33, 'f': 33},
+ {'0': 34, '1': 34, '2': 34, '3': 34,
+ '4': 34, '5': 34, '6': 34, '7': 34,
+ '8': 34, '9': 34, 'A': 34, 'B': 34,
+ 'C': 34, 'D': 34, 'E': 34, 'F': 34,
+ 'a': 34, 'b': 34, 'c': 34, 'd': 34,
+ 'e': 34, 'f': 34},
# 20
- {'0': 34, '1': 34, '2': 34, '3': 34,
- '4': 34, '5': 34, '6': 34, '7': 34},
+ {'0': 35, '1': 35, '2': 35, '3': 35,
+ '4': 35, '5': 35, '6': 35, '7': 35},
# 21
- {'0': 35, '1': 35},
+ {'0': 36, '1': 36},
# 22
- {'.': 23, '0': 22, '1': 22, '2': 22,
- '3': 22, '4': 22, '5': 22, '6': 22,
- '7': 22, '8': 22, '9': 22, 'E': 24,
- 'J': 13, 'e': 24, 'j': 13},
+ {'.': 24, '0': 22, '1': 23, '2': 23,
+ '3': 23, '4': 23, '5': 23, '6': 23,
+ '7': 23, '8': 23, '9': 23, 'E': 25,
+ 'J': 13, 'e': 25, 'j': 13},
# 23
- {'0': 23, '1': 23, '2': 23, '3': 23,
- '4': 23, '5': 23, '6': 23, '7': 23,
- '8': 23, '9': 23, 'E': 36, 'J': 13,
- 'e': 36, 'j': 13},
+ {'.': 24, '0': 23, '1': 23, '2': 23,
+ '3': 23, '4': 23, '5': 23, '6': 23,
+ '7': 23, '8': 23, '9': 23, 'E': 25,
+ 'J': 13, 'e': 25, 'j': 13},
# 24
- {'+': 37, '-': 37, '0': 38, '1': 38,
- '2': 38, '3': 38, '4': 38, '5': 38,
- '6': 38, '7': 38, '8': 38, '9': 38},
+ {'0': 24, '1': 24, '2': 24, '3': 24,
+ '4': 24, '5': 24, '6': 24, '7': 24,
+ '8': 24, '9': 24, 'E': 37, 'J': 13,
+ 'e': 37, 'j': 13},
# 25
- {'0': 25, '1': 25, '2': 25, '3': 25,
- '4': 25, '5': 25, '6': 25, '7': 25,
- '8': 25, '9': 25, 'E': 36, 'J': 13,
- 'e': 36, 'j': 13},
+ {'+': 38, '-': 38, '0': 39, '1': 39,
+ '2': 39, '3': 39, '4': 39, '5': 39,
+ '6': 39, '7': 39, '8': 39, '9': 39},
# 26
+ {'0': 26, '1': 26, '2': 26, '3': 26,
+ '4': 26, '5': 26, '6': 26, '7': 26,
+ '8': 26, '9': 26, 'E': 37, 'J': 13,
+ 'e': 37, 'j': 13},
+ # 27
{},
- # 27
+ # 28
{"'": 13},
- # 28
- {automata.DEFAULT: 39, '\n': 13, '\r': 14},
# 29
- {automata.DEFAULT: 29, '\n': 26,
- '\r': 26, "'": 13, '\\': 28},
+ {automata.DEFAULT: 40, '\n': 13, '\r': 14},
# 30
+ {automata.DEFAULT: 30, '\n': 27,
+ '\r': 27, "'": 13, '\\': 29},
+ # 31
{'"': 13},
- # 31
- {automata.DEFAULT: 40, '\n': 13, '\r': 14},
# 32
- {automata.DEFAULT: 32, '\n': 26,
- '\r': 26, '"': 13, '\\': 31},
+ {automata.DEFAULT: 41, '\n': 13, '\r': 14},
# 33
- {'0': 33, '1': 33, '2': 33, '3': 33,
- '4': 33, '5': 33, '6': 33, '7': 33,
- '8': 33, '9': 33, 'A': 33, 'B': 33,
- 'C': 33, 'D': 33, 'E': 33, 'F': 33,
- 'a': 33, 'b': 33, 'c': 33, 'd': 33,
- 'e': 33, 'f': 33},
+ {automata.DEFAULT: 33, '\n': 27,
+ '\r': 27, '"': 13, '\\': 32},
# 34
{'0': 34, '1': 34, '2': 34, '3': 34,
- '4': 34, '5': 34, '6': 34, '7': 34},
+ '4': 34, '5': 34, '6': 34, '7': 34,
+ '8': 34, '9': 34, 'A': 34, 'B': 34,
+ 'C': 34, 'D': 34, 'E': 34, 'F': 34,
+ 'a': 34, 'b': 34, 'c': 34, 'd': 34,
+ 'e': 34, 'f': 34},
# 35
- {'0': 35, '1': 35},
+ {'0': 35, '1': 35, '2': 35, '3': 35,
+ '4': 35, '5': 35, '6': 35, '7': 35},
# 36
- {'+': 41, '-': 41, '0': 42, '1': 42,
- '2': 42, '3': 42, '4': 42, '5': 42,
- '6': 42, '7': 42, '8': 42, '9': 42},
+ {'0': 36, '1': 36},
# 37
- {'0': 38, '1': 38, '2': 38, '3': 38,
- '4': 38, '5': 38, '6': 38, '7': 38,
- '8': 38, '9': 38},
+ {'+': 42, '-': 42, '0': 43, '1': 43,
+ '2': 43, '3': 43, '4': 43, '5': 43,
+ '6': 43, '7': 43, '8': 43, '9': 43},
# 38
- {'0': 38, '1': 38, '2': 38, '3': 38,
- '4': 38, '5': 38, '6': 38, '7': 38,
- '8': 38, '9': 38, 'J': 13, 'j': 13},
+ {'0': 39, '1': 39, '2': 39, '3': 39,
+ '4': 39, '5': 39, '6': 39, '7': 39,
+ '8': 39, '9': 39},
# 39
- {automata.DEFAULT: 39, '\n': 26,
- '\r': 26, "'": 13, '\\': 28},
+ {'0': 39, '1': 39, '2': 39, '3': 39,
+ '4': 39, '5': 39, '6': 39, '7': 39,
+ '8': 39, '9': 39, 'J': 13, 'j': 13},
# 40
- {automata.DEFAULT: 40, '\n': 26,
- '\r': 26, '"': 13, '\\': 31},
+ {automata.DEFAULT: 40, '\n': 27,
+ '\r': 27, "'": 13, '\\': 29},
# 41
- {'0': 42, '1': 42, '2': 42, '3': 42,
- '4': 42, '5': 42, '6': 42, '7': 42,
- '8': 42, '9': 42},
+ {automata.DEFAULT: 41, '\n': 27,
+ '\r': 27, '"': 13, '\\': 32},
# 42
- {'0': 42, '1': 42, '2': 42, '3': 42,
- '4': 42, '5': 42, '6': 42, '7': 42,
- '8': 42, '9': 42, 'J': 13, 'j': 13},
+ {'0': 43, '1': 43, '2': 43, '3': 43,
+ '4': 43, '5': 43, '6': 43, '7': 43,
+ '8': 43, '9': 43},
+ # 43
+ {'0': 43, '1': 43, '2': 43, '3': 43,
+ '4': 43, '5': 43, '6': 43, '7': 43,
+ '8': 43, '9': 43, 'J': 13, 'j': 13},
]
pseudoDFA = automata.DFA(states, accepts)
More information about the pypy-commit
mailing list