[Python-checkins] r72408 - in python/branches/py3k: Lib/test/test_codecs.py Misc/NEWS Modules/_codecsmodule.c

[walter.doerwald]

Author: walter.doerwald
Date: Wed May  6 16:41:26 2009
New Revision: 72408

Log:
Merged revisions 72404-72406 via svnmerge from 
svn+ssh://pythondev@svn.python.org/python/trunk

........
  r72404 | walter.doerwald | 2009-05-06 16:28:24 +0200 (Mi, 06 Mai 2009) | 3 lines
  
  Issue 3739: The unicode-internal encoder now reports the number of *characters*
  consumed like any other encoder (instead of the number of bytes).
........
  r72406 | walter.doerwald | 2009-05-06 16:32:35 +0200 (Mi, 06 Mai 2009) | 2 lines
  
  Add NEWS entry about issue #3739.
........


Modified:
   python/branches/py3k/   (props changed)
   python/branches/py3k/Lib/test/test_codecs.py
   python/branches/py3k/Misc/NEWS
   python/branches/py3k/Modules/_codecsmodule.c

Modified: python/branches/py3k/Lib/test/test_codecs.py
==============================================================================
--- python/branches/py3k/Lib/test/test_codecs.py	(original)
+++ python/branches/py3k/Lib/test/test_codecs.py	Wed May  6 16:41:26 2009
@@ -872,6 +872,12 @@
                               "UnicodeInternalTest")
             self.assertEquals(("ab", 12), ignored)
 
+    def test_encode_length(self):
+        # Issue 3739
+        encoder = codecs.getencoder("unicode_internal")
+        self.assertEquals(encoder("a")[1], 1)
+        self.assertEquals(encoder("\xe9\u0142")[1], 2)
+
 # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
 nameprep_tests = [
     # 3.1 Map to nothing.
@@ -1317,8 +1323,7 @@
                 name = "latin_1"
             self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
             (b, size) = codecs.getencoder(encoding)(s)
-            if encoding != "unicode_internal":
-                self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
+            self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
             (chars, size) = codecs.getdecoder(encoding)(b)
             self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))
 

Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS	(original)
+++ python/branches/py3k/Misc/NEWS	Wed May  6 16:41:26 2009
@@ -127,6 +127,9 @@
 - Issue #1113244: Py_XINCREF, Py_DECREF, Py_XDECREF: Add `do { ... } while (0)'
   to avoid compiler warnings.
 
+- Issue #3739: The unicode-internal encoder now reports the number of characters
+  consumed like any other encoder (instead of the number of bytes).
+
 Installation
 ------------
 

Modified: python/branches/py3k/Modules/_codecsmodule.c
==============================================================================
--- python/branches/py3k/Modules/_codecsmodule.c	(original)
+++ python/branches/py3k/Modules/_codecsmodule.c	Wed May  6 16:41:26 2009
@@ -669,7 +669,8 @@
     if (PyUnicode_Check(obj)) {
 	data = PyUnicode_AS_DATA(obj);
 	size = PyUnicode_GET_DATA_SIZE(obj);
-	return codec_tuple(PyBytes_FromStringAndSize(data, size), size);
+	return codec_tuple(PyBytes_FromStringAndSize(data, size),
+	                   PyUnicode_GET_SIZE(obj));
     }
     else {
 	if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))