[pypy-svn] r75505 - in pypy/branch/fast-forward/pypy/interpreter: astcompiler pyparser pyparser/test test

[benjamin at codespeak.net]

Author: benjamin
Date: Tue Jun 22 18:52:51 2010
New Revision: 75505

Modified:
   pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py
   pypy/branch/fast-forward/pypy/interpreter/astcompiler/consts.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py
   pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py
   pypy/branch/fast-forward/pypy/interpreter/test/test_syntax.py
Log:
add from __future__ import unicode_literals support

Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/astbuilder.py	Tue Jun 22 18:52:51 2010
@@ -1,4 +1,4 @@
-from pypy.interpreter.astcompiler import ast, misc
+from pypy.interpreter.astcompiler import ast, consts, misc
 from pypy.interpreter.astcompiler import asthelpers # Side effects
 from pypy.interpreter import error
 from pypy.interpreter.pyparser.pygram import syms, tokens
@@ -1060,7 +1060,10 @@
         elif first_child_type == tokens.STRING:
             space = self.space
             encoding = self.compile_info.encoding
-            sub_strings_w = [parsestring.parsestr(space, encoding, s.value)
+            flags = self.compile_info.flags
+            unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS
+            sub_strings_w = [parsestring.parsestr(space, encoding, s.value,
+                                                  unicode_literals)
                              for s in atom_node.children]
             # This implements implicit string concatenation.
             if len(sub_strings_w) > 1:

Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/consts.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/astcompiler/consts.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/consts.py	Tue Jun 22 18:52:51 2010
@@ -16,6 +16,7 @@
 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000
 CO_FUTURE_WITH_STATEMENT = 0x8000
 CO_FUTURE_PRINT_FUNCTION = 0x10000
+CO_FUTURE_UNICODE_LITERALS = 0x20000
 
 PyCF_SOURCE_IS_UTF8 = 0x0100
 PyCF_DONT_IMPLY_DEDENT = 0x0200

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/parsestring.py	Tue Jun 22 18:52:51 2010
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter import unicodehelper
 
-def parsestr(space, encoding, s):
+def parsestr(space, encoding, s, unicode_literals=False):
     # compiler.transformer.Transformer.decode_literal depends on what 
     # might seem like minor details of this function -- changes here 
     # must be reflected there.
@@ -11,7 +11,7 @@
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = False
+    unicode = unicode_literals
 
     # string decoration handling
     o = ord(quote)
@@ -20,6 +20,7 @@
         if quote == 'b' or quote == 'B':
             ps += 1
             quote = s[ps]
+            unicode = False
         elif quote == 'u' or quote == 'U':
             ps += 1
             quote = s[ps]

Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_parsestring.py	Tue Jun 22 18:52:51 2010
@@ -69,6 +69,15 @@
         ret = space.unwrap(w_ret)
         assert ret == eval("# -*- coding: koi8-u -*-\nu'\x81'")
 
+    def test_unicode_literals(self):
+        space = self.space
+        w_ret = parsestring.parsestr(space, None, repr("hello"), True)
+        assert space.isinstance_w(w_ret, space.w_unicode)
+        w_ret = parsestring.parsestr(space, None, "b'hi'", True)
+        assert space.isinstance_w(w_ret, space.w_str)
+        w_ret = parsestring.parsestr(space, None, "r'hi'", True)
+        assert space.isinstance_w(w_ret, space.w_unicode)
+
     def test_bytes(self):
         space = self.space
         b = "b'hello'"

Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_syntax.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/test/test_syntax.py	(original)
+++ pypy/branch/fast-forward/pypy/interpreter/test/test_syntax.py	Tue Jun 22 18:52:51 2010
@@ -336,6 +336,25 @@
         exec s in ns
         assert ns["s"].getvalue() == "Hello, person\n"
 
+
+class AppTestUnicodeLiterals:
+
+    def test_simple(self):
+        s = """from __future__ import unicode_literals
+x = 'u'
+y = r'u'
+z = u'u'
+b = b'u'
+c = br'u'"""
+        ns = {}
+        exec s in ns
+        assert isinstance(ns["x"], unicode)
+        assert isinstance(ns["y"], unicode)
+        assert isinstance(ns["z"], unicode)
+        assert isinstance(ns["b"], str)
+        assert isinstance(ns["c"], str)
+
+
 class AppTestWith:
     def test_with_simple(self):