[Python-checkins] r55078 - in sandbox/trunk/2to3: Grammar.txt pgen2/tokenize.py

[guido.van.rossum]

Author: guido.van.rossum
Date: Wed May  2 21:02:07 2007
New Revision: 55078

Modified:
   sandbox/trunk/2to3/Grammar.txt
   sandbox/trunk/2to3/pgen2/tokenize.py
Log:
Hack the grammar to support nonlocal.
Add b"..." support to the tokenizer.


Modified: sandbox/trunk/2to3/Grammar.txt
==============================================================================
--- sandbox/trunk/2to3/Grammar.txt	(original)
+++ sandbox/trunk/2to3/Grammar.txt	Wed May  2 21:02:07 2007
@@ -76,7 +76,7 @@
 import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
-global_stmt: 'global' NAME (',' NAME)*
+global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 

Modified: sandbox/trunk/2to3/pgen2/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/tokenize.py	(original)
+++ sandbox/trunk/2to3/pgen2/tokenize.py	Wed May  2 21:02:07 2007
@@ -87,9 +87,9 @@
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.
-ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
-                r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
@@ -100,13 +100,21 @@
             "'''": single3prog, '"""': double3prog,
             "r'''": single3prog, 'r"""': double3prog,
             "u'''": single3prog, 'u"""': double3prog,
+            "b'''": single3prog, 'b"""': double3prog,
             "ur'''": single3prog, 'ur"""': double3prog,
+            "br'''": single3prog, 'br"""': double3prog,
             "R'''": single3prog, 'R"""': double3prog,
             "U'''": single3prog, 'U"""': double3prog,
+            "B'''": single3prog, 'B"""': double3prog,
             "uR'''": single3prog, 'uR"""': double3prog,
             "Ur'''": single3prog, 'Ur"""': double3prog,
             "UR'''": single3prog, 'UR"""': double3prog,
-            'r': None, 'R': None, 'u': None, 'U': None}
+            "bR'''": single3prog, 'bR"""': double3prog,
+            "Br'''": single3prog, 'Br"""': double3prog,
+            "BR'''": single3prog, 'BR"""': double3prog,
+            'r': None, 'R': None,
+            'u': None, 'U': None,
+            'b': None, 'B': None}
 
 triple_quoted = {}
 for t in ("'''", '"""',