[Python-checkins] cpython (merge 3.2 -> default): Issue #15379: Fix passing of non-BMP characters as integers for the charmap

antoine.pitrou python-checkins at python.org
Sun Sep 23 20:01:15 CEST 2012


http://hg.python.org/cpython/rev/c64dec45d46f
changeset:   79108:c64dec45d46f
parent:      79106:d81b9e46c06b
parent:      79107:620d23f7ad41
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Sun Sep 23 20:00:04 2012 +0200
summary:
  Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings).
Patch by Serhiy Storchaka.

files:
  Lib/test/test_codecs.py |  116 ++++++++++++++++++++++++++++
  Misc/NEWS               |    3 +
  Objects/unicodeobject.c |    7 +-
  3 files changed, 123 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1693,6 +1693,15 @@
         )
 
         self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
+            ("\U0010FFFFbc", 3)
+        )
+
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
+        )
+
+        self.assertEqual(
             codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
             ("ab\ufffd", 3)
         )
@@ -1718,6 +1727,113 @@
             ("", len(allbytes))
         )
 
+    def test_decode_with_int2str_map(self):
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: 'a', 1: 'b', 2: 'c'}),
+            ("abc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: 'Aa', 1: 'Bb', 2: 'Cc'}),
+            ("AaBbCc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: '\U0010FFFF', 1: 'b', 2: 'c'}),
+            ("\U0010FFFFbc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: 'a', 1: 'b', 2: ''}),
+            ("ab", 3)
+        )
+
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, b"\x00\x01\x02", "strict",
+                                   {0: 'a', 1: 'b'}
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "replace",
+                                  {0: 'a', 1: 'b'}),
+            ("ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "replace",
+                                  {0: 'a', 1: 'b', 2: None}),
+            ("ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+                                  {0: 'a', 1: 'b'}),
+            ("ab", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+                                  {0: 'a', 1: 'b', 2: None}),
+            ("ab", 3)
+        )
+
+        allbytes = bytes(range(256))
+        self.assertEqual(
+            codecs.charmap_decode(allbytes, "ignore", {}),
+            ("", len(allbytes))
+        )
+
+    def test_decode_with_int2int_map(self):
+        a = ord('a')
+        b = ord('b')
+        c = ord('c')
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: a, 1: b, 2: c}),
+            ("abc", 3)
+        )
+
+        # Issue #15379
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: 0x10FFFF, 1: b, 2: c}),
+            ("\U0010FFFFbc", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "strict",
+                                  {0: sys.maxunicode, 1: b, 2: c}),
+            (chr(sys.maxunicode) + "bc", 3)
+        )
+
+        self.assertRaises(TypeError,
+            codecs.charmap_decode, b"\x00\x01\x02", "strict",
+                                   {0: sys.maxunicode + 1, 1: b, 2: c}
+        )
+
+        self.assertRaises(UnicodeDecodeError,
+            codecs.charmap_decode, b"\x00\x01\x02", "strict",
+                                   {0: a, 1: b},
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "replace",
+                                  {0: a, 1: b}),
+            ("ab\ufffd", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+                                  {0: a, 1: b}),
+            ("ab", 3)
+        )
+
+
 class WithStmtTest(unittest.TestCase):
     def test_encodedfile(self):
         f = io.BytesIO(b"\xc3\xbc")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #15379: Fix passing of non-BMP characters as integers for the charmap
+  decoder (already working as unicode strings).  Patch by Serhiy Storchaka.
+
 - Issue #15144: Fix possible integer overflow when handling pointers as
   integer values, by using Py_uintptr_t instead of size_t.  Patch by
   Serhiy Storchaka.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7525,9 +7525,10 @@
             /* Apply mapping */
             if (PyLong_Check(x)) {
                 long value = PyLong_AS_LONG(x);
-                if (value < 0 || value > 65535) {
-                    PyErr_SetString(PyExc_TypeError,
-                                    "character mapping must be in range(65536)");
+                if (value < 0 || value > MAX_UNICODE) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "character mapping must be in range(0x%lx)",
+                                 (unsigned long)MAX_UNICODE + 1);
                     Py_DECREF(x);
                     goto onError;
                 }

-- 
Repository URL: http://hg.python.org/cpython


More information about the Python-checkins mailing list