Mailman 3 August 2005 - Python-checkins

python/dist/src/Include patchlevel.h, 2.83.2.6, 2.83.2.7
by anthonybaxter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Include In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv30627/Include Modified Files: Tag: release24-maint patchlevel.h Log Message: Make testing a bit easier Index: patchlevel.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/patchlevel.h,v retrieving revision 2.83.2.6 retrieving revision 2.83.2.7 diff -u -d -r2.83.2.6 -r2.83.2.7 --- patchlevel.h 29 Mar 2005 12:33:33 -0000 2.83.2.6 +++ patchlevel.h 30 Aug 2005 12:26:43 -0000 2.83.2.7 @@ -26,7 +26,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.4.1" +#define PY_VERSION "2.4.1+" /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */

1 0

python/dist/src/Objects unicodeobject.c, 2.230, 2.230.2.1
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Objects In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5498/Objects Modified Files: Tag: release24-maint unicodeobject.c Log Message: Backport checkin: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the patch by Nik Haldimann that detects truncated data) Index: unicodeobject.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Objects/unicodeobject.c,v retrieving revision 2.230 retrieving revision 2.230.2.1 diff -u -d -r2.230 -r2.230.2.1 --- unicodeobject.c 22 Nov 2004 13:02:30 -0000 2.230 +++ unicodeobject.c 30 Aug 2005 10:46:06 -0000 2.230.2.1 @@ -2273,6 +2273,81 @@ PyUnicode_GET_SIZE(unicode)); } +/* --- Unicode Internal Codec ------------------------------------------- */ + +PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s, + int size, + const char *errors) +{ + const char *starts = s; + int startinpos; + int endinpos; + int outpos; + Py_UNICODE unimax; + PyUnicodeObject *v; + Py_UNICODE *p; + const char *end; + const char *reason; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + + unimax = PyUnicode_GetMax(); + v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE); + if (v == NULL) + goto onError; + if (PyUnicode_GetSize((PyObject *)v) == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); + end = s + size; + + while (s < end) { + *p = *(Py_UNICODE *)s; + /* We have to sanity check the raw data, otherwise doom looms for + some malformed UCS-4 data. */ + if ( + #ifdef Py_UNICODE_WIDE + *p > unimax || *p < 0 || + #endif + end-s < Py_UNICODE_SIZE + ) + { + startinpos = s - starts; + if (end-s < Py_UNICODE_SIZE) { + endinpos = end-starts; + reason = "truncated input"; + } + else { + endinpos = s - starts + Py_UNICODE_SIZE; + reason = "illegal code point (> 0x10FFFF)"; + } + outpos = p - PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicode_internal", reason, + starts, size, &startinpos, &endinpos, &exc, &s, + (PyObject **)&v, &outpos, &p)) { + goto onError; + } + } + else { + p++; + s += Py_UNICODE_SIZE; + } + } + + if (_PyUnicode_Resize(&v, (int)(p - PyUnicode_AS_UNICODE(v))) < 0) + goto onError; + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return (PyObject *)v; + + onError: + Py_XDECREF(v); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return NULL; +} + /* --- Latin-1 Codec ------------------------------------------------------ */ PyObject *PyUnicode_DecodeLatin1(const char *s,

1 0

python/dist/src/Include unicodeobject.h, 2.48, 2.48.2.1
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Include In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5498/Include Modified Files: Tag: release24-maint unicodeobject.h Log Message: Backport checkin: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the patch by Nik Haldimann that detects truncated data) Index: unicodeobject.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/unicodeobject.h,v retrieving revision 2.48 retrieving revision 2.48.2.1 diff -u -d -r2.48 -r2.48.2.1 --- unicodeobject.h 22 Nov 2004 13:02:29 -0000 2.48 +++ unicodeobject.h 30 Aug 2005 10:46:05 -0000 2.48.2.1 @@ -797,6 +797,16 @@ int length /* Number of Py_UNICODE chars to encode */ ); +/* --- Unicode Internal Codec --------------------------------------------- + + Only for internal use in _codecsmodule.c */ + +PyObject *_PyUnicode_DecodeUnicodeInternal( + const char *string, + int length, + const char *errors + ); + /* --- Latin-1 Codecs ----------------------------------------------------- Note: Latin-1 corresponds to the first 256 Unicode ordinals.

1 0

python/dist/src/Misc NEWS, 1.1193.2.91, 1.1193.2.92
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Misc In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5498/Misc Modified Files: Tag: release24-maint NEWS Log Message: Backport checkin: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the patch by Nik Haldimann that detects truncated data) Index: NEWS =================================================================== RCS file: /cvsroot/python/python/dist/src/Misc/NEWS,v retrieving revision 1.1193.2.91 retrieving revision 1.1193.2.92 diff -u -d -r1.1193.2.91 -r1.1193.2.92 --- NEWS 27 Aug 2005 19:29:21 -0000 1.1193.2.91 +++ NEWS 30 Aug 2005 10:46:06 -0000 1.1193.2.92 @@ -137,6 +137,10 @@ line ending. Remove the special handling of a "\r\n" that has been split between two lines. +- Bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain + about illegal code points. The codec now supports PEP 293 style error + handlers. + Build -----

1 0

python/dist/src/Lib/test test_codeccallbacks.py, 1.16.4.1, 1.16.4.2 test_codecs.py, 1.15.2.7, 1.15.2.8
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Lib/test In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5498/Lib/test Modified Files: Tag: release24-maint test_codeccallbacks.py test_codecs.py Log Message: Backport checkin: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the patch by Nik Haldimann that detects truncated data) Index: test_codeccallbacks.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_codeccallbacks.py,v retrieving revision 1.16.4.1 retrieving revision 1.16.4.2 diff -u -d -r1.16.4.1 -r1.16.4.2 --- test_codeccallbacks.py 14 Dec 2004 21:33:13 -0000 1.16.4.1 +++ test_codeccallbacks.py 30 Aug 2005 10:46:06 -0000 1.16.4.2 @@ -111,7 +111,7 @@ sout += "\\U%08x" % sys.maxunicode self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) - def test_relaxedutf8(self): + def test_decoderelaxedutf8(self): # This is the test for a decoding callback handler, # that relaxes the UTF-8 minimal encoding restriction. # A null byte that is encoded as "\xc0\x80" will be @@ -158,6 +158,35 @@ charmap[ord("?")] = u"XYZ" self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) + def test_decodeunicodeinternal(self): + self.assertRaises( + UnicodeDecodeError, + "\x00\x00\x00\x00\x00".decode, + "unicode-internal", + ) + if sys.maxunicode > 0xffff: + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + return (u"\x01", 1) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), + u"\u0000" + ) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"), + u"\u0000\ufffd" + ) + + codecs.register_error("test.hui", handler_unicodeinternal) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"), + u"\u0000\u0001\u0000" + ) + def test_callbacks(self): def handler1(exc): if not isinstance(exc, UnicodeEncodeError) \ @@ -503,7 +532,8 @@ for (enc, bytes) in ( ("ascii", "\xff"), ("utf-8", "\xff"), - ("utf-7", "+x-") + ("utf-7", "+x-"), + ("unicode-internal", "\x00"), ): self.assertRaises( TypeError, Index: test_codecs.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_codecs.py,v retrieving revision 1.15.2.7 retrieving revision 1.15.2.8 diff -u -d -r1.15.2.7 -r1.15.2.8 --- test_codecs.py 25 Aug 2005 11:04:04 -0000 1.15.2.7 +++ test_codecs.py 30 Aug 2005 10:46:06 -0000 1.15.2.8 @@ -1,7 +1,7 @@ from test import test_support import unittest import codecs -import StringIO +import sys, StringIO class Queue(object): """ @@ -455,6 +455,54 @@ for uni, puny in punycode_testcases: self.assertEquals(uni, puny.decode("punycode")) +class UnicodeInternalTest(unittest.TestCase): + def test_bug1251300(self): + # Decoding with unicode_internal used to not correctly handle "code + # points" above 0x10ffff on UCS-4 builds. + if sys.maxunicode > 0xffff: + ok = [ + ("\x00\x10\xff\xff", u"\U0010ffff"), + ("\x00\x00\x01\x01", u"\U00000101"), + ("", u""), + ] + not_ok = [ + "\x7f\xff\xff\xff", + "\x80\x00\x00\x00", + "\x81\x00\x00\x00", + "\x00", + "\x00\x00\x00\x00\x00", + ] + for internal, uni in ok: + if sys.byteorder == "little": + internal = "".join(reversed(internal)) + self.assertEquals(uni, internal.decode("unicode_internal")) + for internal in not_ok: + if sys.byteorder == "little": + internal = "".join(reversed(internal)) + self.assertRaises(UnicodeDecodeError, internal.decode, + "unicode_internal") + + def test_decode_error_attributes(self): + if sys.maxunicode > 0xffff: + try: + "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") + except UnicodeDecodeError, ex: + self.assertEquals("unicode_internal", ex.encoding) + self.assertEquals("\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) + self.assertEquals(4, ex.start) + self.assertEquals(8, ex.end) + else: + self.fail() + + def test_decode_callback(self): + if sys.maxunicode > 0xffff: + codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) + decoder = codecs.getdecoder("unicode_internal") + ab = u"ab".encode("unicode_internal") + ignored = decoder("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), + "UnicodeInternalTest") + self.assertEquals((u"ab", 12), ignored) + # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ # 3.1 Map to nothing. @@ -696,6 +744,7 @@ EscapeDecodeTest, RecodingTest, PunycodeTest, + UnicodeInternalTest, NameprepTest, CodecTest, CodecsModuleTest,

1 0

python/dist/src/Modules _codecsmodule.c, 2.20.2.1, 2.20.2.2
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Modules In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5498/Modules Modified Files: Tag: release24-maint _codecsmodule.c Log Message: Backport checkin: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the patch by Nik Haldimann that detects truncated data) Index: _codecsmodule.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/_codecsmodule.c,v retrieving revision 2.20.2.1 retrieving revision 2.20.2.2 diff -u -d -r2.20.2.1 -r2.20.2.2 --- _codecsmodule.c 8 Mar 2005 15:05:17 -0000 2.20.2.1 +++ _codecsmodule.c 30 Aug 2005 10:46:06 -0000 2.20.2.2 @@ -254,8 +254,8 @@ else { if (PyObject_AsReadBuffer(obj, (const void **)&data, &size)) return NULL; - return codec_tuple(PyUnicode_FromUnicode((Py_UNICODE *)data, - size / sizeof(Py_UNICODE)), + + return codec_tuple(_PyUnicode_DecodeUnicodeInternal(data, size, errors), size); } }

1 0

python/dist/src/Misc NEWS,1.1357,1.1358
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Misc In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv1718/Misc Modified Files: NEWS Log Message: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the Nik Haldimann's patch that detects truncated data) Index: NEWS =================================================================== RCS file: /cvsroot/python/python/dist/src/Misc/NEWS,v retrieving revision 1.1357 retrieving revision 1.1358 diff -u -d -r1.1357 -r1.1358 --- NEWS 27 Aug 2005 10:07:56 -0000 1.1357 +++ NEWS 30 Aug 2005 10:23:13 -0000 1.1358 @@ -435,6 +435,10 @@ line ending. Remove the special handling of a "\r\n" that has been split between two lines. +- Bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain + about illegal code points. The codec now supports PEP 293 style error + handlers. + Build -----

1 0

python/dist/src/Modules _codecsmodule.c, 2.21, 2.22
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Modules In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv1718/Modules Modified Files: _codecsmodule.c Log Message: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the Nik Haldimann's patch that detects truncated data) Index: _codecsmodule.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/_codecsmodule.c,v retrieving revision 2.21 retrieving revision 2.22 diff -u -d -r2.21 -r2.22 --- _codecsmodule.c 8 Mar 2005 15:03:08 -0000 2.21 +++ _codecsmodule.c 30 Aug 2005 10:23:14 -0000 2.22 @@ -254,8 +254,8 @@ else { if (PyObject_AsReadBuffer(obj, (const void **)&data, &size)) return NULL; - return codec_tuple(PyUnicode_FromUnicode((Py_UNICODE *)data, - size / sizeof(Py_UNICODE)), + + return codec_tuple(_PyUnicode_DecodeUnicodeInternal(data, size, errors), size); } }

1 0

python/dist/src/Include unicodeobject.h, 2.48, 2.49
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Include In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv1718/Include Modified Files: unicodeobject.h Log Message: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the Nik Haldimann's patch that detects truncated data) Index: unicodeobject.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/unicodeobject.h,v retrieving revision 2.48 retrieving revision 2.49 diff -u -d -r2.48 -r2.49 --- unicodeobject.h 22 Nov 2004 13:02:29 -0000 2.48 +++ unicodeobject.h 30 Aug 2005 10:23:13 -0000 2.49 @@ -797,6 +797,16 @@ int length /* Number of Py_UNICODE chars to encode */ ); +/* --- Unicode Internal Codec --------------------------------------------- + + Only for internal use in _codecsmodule.c */ + +PyObject *_PyUnicode_DecodeUnicodeInternal( + const char *string, + int length, + const char *errors + ); + /* --- Latin-1 Codecs ----------------------------------------------------- Note: Latin-1 corresponds to the first 256 Unicode ordinals.

1 0

python/dist/src/Lib/test test_codeccallbacks.py, 1.17, 1.18 test_codecs.py, 1.25, 1.26
by doerwalter＠users.sourceforge.net 30 Aug '05

30 Aug '05

Update of /cvsroot/python/python/dist/src/Lib/test In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv1718/Lib/test Modified Files: test_codeccallbacks.py test_codecs.py Log Message: SF bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain about illegal code points. The codec now supports PEP 293 style error handlers. (This is a variant of the Nik Haldimann's patch that detects truncated data) Index: test_codeccallbacks.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_codeccallbacks.py,v retrieving revision 1.17 retrieving revision 1.18 diff -u -d -r1.17 -r1.18 --- test_codeccallbacks.py 14 Dec 2004 21:28:07 -0000 1.17 +++ test_codeccallbacks.py 30 Aug 2005 10:23:13 -0000 1.18 @@ -111,7 +111,7 @@ sout += "\\U%08x" % sys.maxunicode self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) - def test_relaxedutf8(self): + def test_decoderelaxedutf8(self): # This is the test for a decoding callback handler, # that relaxes the UTF-8 minimal encoding restriction. # A null byte that is encoded as "\xc0\x80" will be @@ -158,6 +158,35 @@ charmap[ord("?")] = u"XYZ" self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) + def test_decodeunicodeinternal(self): + self.assertRaises( + UnicodeDecodeError, + "\x00\x00\x00\x00\x00".decode, + "unicode-internal", + ) + if sys.maxunicode > 0xffff: + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + return (u"\x01", 1) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), + u"\u0000" + ) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"), + u"\u0000\ufffd" + ) + + codecs.register_error("test.hui", handler_unicodeinternal) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"), + u"\u0000\u0001\u0000" + ) + def test_callbacks(self): def handler1(exc): if not isinstance(exc, UnicodeEncodeError) \ @@ -503,7 +532,8 @@ for (enc, bytes) in ( ("ascii", "\xff"), ("utf-8", "\xff"), - ("utf-7", "+x-") + ("utf-7", "+x-"), + ("unicode-internal", "\x00"), ): self.assertRaises( TypeError, Index: test_codecs.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_codecs.py,v retrieving revision 1.25 retrieving revision 1.26 diff -u -d -r1.25 -r1.26 --- test_codecs.py 25 Aug 2005 11:03:38 -0000 1.25 +++ test_codecs.py 30 Aug 2005 10:23:13 -0000 1.26 @@ -1,7 +1,7 @@ from test import test_support import unittest import codecs -import StringIO +import sys, StringIO class Queue(object): """ @@ -453,6 +453,54 @@ for uni, puny in punycode_testcases: self.assertEquals(uni, puny.decode("punycode")) +class UnicodeInternalTest(unittest.TestCase): + def test_bug1251300(self): + # Decoding with unicode_internal used to not correctly handle "code + # points" above 0x10ffff on UCS-4 builds. + if sys.maxunicode > 0xffff: + ok = [ + ("\x00\x10\xff\xff", u"\U0010ffff"), + ("\x00\x00\x01\x01", u"\U00000101"), + ("", u""), + ] + not_ok = [ + "\x7f\xff\xff\xff", + "\x80\x00\x00\x00", + "\x81\x00\x00\x00", + "\x00", + "\x00\x00\x00\x00\x00", + ] + for internal, uni in ok: + if sys.byteorder == "little": + internal = "".join(reversed(internal)) + self.assertEquals(uni, internal.decode("unicode_internal")) + for internal in not_ok: + if sys.byteorder == "little": + internal = "".join(reversed(internal)) + self.assertRaises(UnicodeDecodeError, internal.decode, + "unicode_internal") + + def test_decode_error_attributes(self): + if sys.maxunicode > 0xffff: + try: + "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") + except UnicodeDecodeError, ex: + self.assertEquals("unicode_internal", ex.encoding) + self.assertEquals("\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) + self.assertEquals(4, ex.start) + self.assertEquals(8, ex.end) + else: + self.fail() + + def test_decode_callback(self): + if sys.maxunicode > 0xffff: + codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) + decoder = codecs.getdecoder("unicode_internal") + ab = u"ab".encode("unicode_internal") + ignored = decoder("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), + "UnicodeInternalTest") + self.assertEquals((u"ab", 12), ignored) + # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ # 3.1 Map to nothing. @@ -885,6 +933,7 @@ EscapeDecodeTest, RecodingTest, PunycodeTest, + UnicodeInternalTest, NameprepTest, CodecTest, CodecsModuleTest,

1 0