[Python-checkins] cpython (2.7): Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
serhiy.storchaka
python-checkins at python.org
Tue Jan 15 14:39:59 CET 2013
http://hg.python.org/cpython/rev/33a8ef498b1e
changeset: 81517:33a8ef498b1e
branch: 2.7
parent: 81506:da5e520a7ba5
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Jan 15 14:42:59 2013 +0200
summary:
Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
in any mapping, not only in an unicode string.
files:
Lib/test/test_codecs.py | 56 ++++++++++++++++++++++++++--
Misc/NEWS | 3 +
Objects/unicodeobject.c | 46 ++++++++++++----------
3 files changed, 79 insertions(+), 26 deletions(-)
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1551,6 +1551,14 @@
(u"abc", 3)
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab"
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict", u"ab\ufffe"
+ )
+
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace", u"ab"),
(u"ab\ufffd", 3)
@@ -1566,10 +1574,6 @@
(u"ab", 3)
)
- self.assertRaises(UnicodeDecodeError,
- codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab"
- )
-
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
(u"ab", 3)
@@ -1611,6 +1615,17 @@
{0: u'a', 1: u'b'}
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: None}
+ )
+
+ # Issue #14850
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: u'\ufffe'}
+ )
+
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace",
{0: u'a', 1: u'b'}),
@@ -1623,6 +1638,13 @@
(u"ab\ufffd", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b', 2: u'\ufffe'}),
+ (u"ab\ufffd", 3)
+ )
+
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore",
{0: u'a', 1: u'b'}),
@@ -1635,7 +1657,14 @@
(u"ab", 3)
)
- allbytes = bytes(range(256))
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b', 2: u'\ufffe'}),
+ (u"ab", 3)
+ )
+
+ allbytes = "".join(chr(i) for i in xrange(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
(u"", len(allbytes))
@@ -1669,6 +1698,11 @@
{0: a, 1: b},
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: 0xFFFE},
+ )
+
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace",
{0: a, 1: b}),
@@ -1676,11 +1710,23 @@
)
self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore",
{0: a, 1: b}),
(u"ab", 3)
)
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: a, 1: b, 2: 0xFFFE}),
+ (u"ab", 3)
+ )
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -9,6 +9,9 @@
Core and Builtins
-----------------
+- Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
+ in any mapping, not only in an unicode string.
+
- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
Amaury Forgeot d'Arc.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4121,15 +4121,18 @@
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
- x = Py_None;
- Py_INCREF(x);
+ goto Undefined;
} else
goto onError;
}
/* Apply mapping */
+ if (x == Py_None)
+ goto Undefined;
if (PyInt_Check(x)) {
long value = PyInt_AS_LONG(x);
+ if (value == 0xFFFE)
+ goto Undefined;
if (value < 0 || value > 0x10FFFF) {
PyErr_SetString(PyExc_TypeError,
"character mapping must be in range(0x110000)");
@@ -4162,29 +4165,16 @@
#endif
*p++ = (Py_UNICODE)value;
}
- else if (x == Py_None) {
- /* undefined mapping */
- outpos = p-PyUnicode_AS_UNICODE(v);
- startinpos = s-starts;
- endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "charmap", "character maps to <undefined>",
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p)) {
- Py_DECREF(x);
- goto onError;
- }
- Py_DECREF(x);
- continue;
- }
else if (PyUnicode_Check(x)) {
Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
- if (targetsize == 1)
+ if (targetsize == 1) {
/* 1-1 mapping */
- *p++ = *PyUnicode_AS_UNICODE(x);
-
+ Py_UNICODE value = *PyUnicode_AS_UNICODE(x);
+ if (value == 0xFFFE)
+ goto Undefined;
+ *p++ = value;
+ }
else if (targetsize > 1) {
/* 1-n mapping */
if (targetsize > extrachars) {
@@ -4218,6 +4208,20 @@
}
Py_DECREF(x);
++s;
+ continue;
+Undefined:
+ /* undefined mapping */
+ Py_XDECREF(x);
+ outpos = p-PyUnicode_AS_UNICODE(v);
+ startinpos = s-starts;
+ endinpos = startinpos+1;
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "charmap", "character maps to <undefined>",
+ starts, size, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos, &p)) {
+ goto onError;
+ }
}
}
if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list