# [Python-checkins] r54511 - in sandbox/trunk/2to3: pgen2/tokenize.py tests/test_grammar.py

collin.winter python-checkins at python.org
Wed Mar 21 23:37:04 CET 2007

```Author: collin.winter
Date: Wed Mar 21 23:36:59 2007
New Revision: 54511

Modified:
sandbox/trunk/2to3/pgen2/tokenize.py
sandbox/trunk/2to3/tests/test_grammar.py
Log:
Add support for py3k's binary literals.

Modified: sandbox/trunk/2to3/pgen2/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/tokenize.py	(original)
+++ sandbox/trunk/2to3/pgen2/tokenize.py	Wed Mar 21 23:36:59 2007
@@ -46,10 +46,11 @@
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Name = r'[a-zA-Z_]\w*'

+Binnumber = r'0b[01]*'
Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
Octnumber = r'0[o]?[0-7]*[lL]?'
Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+'
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Expfloat = r'\d+' + Exponent

Modified: sandbox/trunk/2to3/tests/test_grammar.py
==============================================================================
--- sandbox/trunk/2to3/tests/test_grammar.py	(original)
+++ sandbox/trunk/2to3/tests/test_grammar.py	Wed Mar 21 23:36:59 2007
@@ -131,6 +131,12 @@
class TestNumericLiterals(GrammarTest):
def test_new_octal_notation(self):
self.validate("""0o7777777777777""")
+
+    def test_new_binary_notation(self):
+        self.validate("""0b101010""")
+        self.invalid_syntax("""0b""")
+        self.invalid_syntax("""0b0101021""")
+        self.invalid_syntax("""b010101""")

class TestGrammarFiles(GrammarTest):
```