[Python-checkins] r57740 - in sandbox/trunk/2to3: pgen2/tokenize.py tests/test_parser.py

[collin.winter]

Author: collin.winter
Date: Thu Aug 30 19:53:12 2007
New Revision: 57740

Modified:
   sandbox/trunk/2to3/   (props changed)
   sandbox/trunk/2to3/pgen2/tokenize.py
   sandbox/trunk/2to3/tests/test_parser.py
Log:
Fix a bug in pgen2.tokenize related to multi-line bytes literals.


Modified: sandbox/trunk/2to3/pgen2/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/tokenize.py	(original)
+++ sandbox/trunk/2to3/pgen2/tokenize.py	Thu Aug 30 19:53:12 2007
@@ -127,8 +127,11 @@
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
+          "b'", 'b"', "B'", 'B"',
           "ur'", 'ur"', "Ur'", 'Ur"',
-          "uR'", 'uR"', "UR'", 'UR"' ):
+          "uR'", 'uR"', "UR'", 'UR"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"', ):
     single_quoted[t] = t
 
 tabsize = 8

Modified: sandbox/trunk/2to3/tests/test_parser.py
==============================================================================
--- sandbox/trunk/2to3/tests/test_parser.py	(original)
+++ sandbox/trunk/2to3/tests/test_parser.py	Thu Aug 30 19:53:12 2007
@@ -155,6 +155,27 @@
                 self.fail("Idempotency failed: %s" % filepath)
 
 
+class TestLiterals(GrammarTest):
+
+    def test_multiline_bytes_literals(self):
+        s = """
+            md5test(b"\xaa" * 80,
+                    (b"Test Using Larger Than Block-Size Key "
+                     b"and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        self.validate(s)
+
+    def test_multiline_str_literals(self):
+        s = """
+            md5test("\xaa" * 80,
+                    ("Test Using Larger Than Block-Size Key "
+                     "and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        self.validate(s)
+
+
 def diff(fn, tree):
     f = open("@", "w")
     try: