[pypy-commit] pypy py3k: fix handling of identifiers in the compiler and the locals' conversions

[pjenvey]

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r64694:e9592b260715
Date: 2013-05-31 13:43 -0700
http://bitbucket.org/pypy/pypy/changeset/e9592b260715/

Log:	fix handling of identifiers in the compiler and the locals'
	conversions

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -305,7 +305,7 @@
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
-                self.load_const(self.space.wrap(kwonly.arg))
+                self.load_const(self.space.wrap(kwonly.arg.decode('utf-8')))
                 default.walkabout(self)
                 defaults += 1
         return defaults
@@ -336,7 +336,8 @@
         if l:
             if l > 65534:
                 self.error("too many annotations", func)
-            w_tup = space.newtuple([space.wrap(name) for name in names])
+            w_tup = space.newtuple([space.wrap(name.decode('utf-8'))
+                                    for name in names])
             self.load_const(w_tup)
             l += 1
         return l
@@ -389,7 +390,7 @@
         # 3. load a function (or closure) made from the code object
         self._make_function(code, 0)
         # 4. load class name
-        self.load_const(self.space.wrap(cls.name))
+        self.load_const(self.space.wrap(cls.name.decode('utf-8')))
         # 5. generate the rest of the code for the call
         self._make_call(2,
                         cls.bases, cls.keywords,
@@ -723,7 +724,7 @@
         for i in range(len(imp.names)):
             alias = imp.names[i]
             assert isinstance(alias, ast.alias)
-            names_w[i] = space.wrap(alias.name)
+            names_w[i] = space.wrap(alias.name.decode('utf-8'))
         self.load_const(space.newtuple(names_w))
         if imp.module:
             mod_name = imp.module
@@ -1024,7 +1025,7 @@
         self.name_op(name.id, name.ctx)
 
     def visit_keyword(self, keyword):
-        self.load_const(self.space.wrap(keyword.arg))
+        self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
         keyword.value.walkabout(self)
 
     def _make_call(self, n, # args already pushed
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -108,7 +108,7 @@
             name = varnames[i]
             w_value = fastscope_w[i]
             if w_value is not None:
-                w_name = self.space.wrap(name)
+                w_name = self.space.wrap(name.decode('utf-8'))
                 self.space.setitem(self.w_locals, w_name, w_value)
 
     def locals2fast(self):
@@ -120,7 +120,7 @@
         new_fastlocals_w = [None] * numlocals
 
         for i in range(min(len(varnames), numlocals)):
-            w_name = self.space.wrap(varnames[i])
+            w_name = self.space.wrap(varnames[i].decode('utf-8'))
             try:
                 w_value = self.space.getitem(self.w_locals, w_name)
             except OperationError, e:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -840,6 +840,10 @@
         raises(SyntaxError, eval, b'\xff\x20')
         raises(SyntaxError, eval, b'\xef\xbb\x20')
 
+    def test_import_nonascii(self):
+        c = compile('from os import 日本', '', 'exec')
+        assert ('日本',) in c.co_consts
+
     def test_cpython_issue2301(self):
         skip('XXX')
         try:
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 import unittest
 from pypy.interpreter import eval
 from pypy.interpreter.function import Function, Method, descr_function_get
@@ -43,6 +44,17 @@
         assert f.__kwdefaults__ is None
         raises(TypeError, f)
         assert f(kw=42) == 42
+        def f(*, 日本=3): return kw
+        assert f.__kwdefaults__ == {"日本" : 3}
+        """
+
+    def test_kw_nonascii(self):
+        """
+        def f(日本: str=1):
+            return 日本
+        assert f.__annotations__ == {'日本': str}
+        assert f() == 1
+        assert f(日本='bar') == 'bar'
         """
 
     def test_code_is_ok(self):