Mailman 3 June 2004 - Python-checkins

python/dist/src/Lib pre.py,1.16,NONE
by akuchling＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Lib In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8765 Removed Files: pre.py Log Message: Remove pre module --- pre.py DELETED ---

1 0

python/dist/src/Misc NEWS,1.983,1.984
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Misc In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv1595/Misc Modified Files: NEWS Log Message: Fix grammar hopefully. :) Index: NEWS =================================================================== RCS file: /cvsroot/python/python/dist/src/Misc/NEWS,v retrieving revision 1.983 retrieving revision 1.984 diff -C2 -d -r1.983 -r1.984 *** NEWS 2 Jun 2004 16:49:09 -0000 1.983 --- NEWS 2 Jun 2004 16:52:49 -0000 1.984 *************** *** 13,18 **** ----------------- ! - Unicode type got two new methods; iswide() and width(). They ! manipulate east asian width information as of Unicode TR11. - Improved the tuple hashing algorithm to give fewer collisions in --- 13,18 ---- ----------------- ! - Unicode type got two new methods now; iswide() and width(). They ! manipulate east asian width information from Unicode TR11. - Improved the tuple hashing algorithm to give fewer collisions in

1 0

python/dist/src/Misc NEWS,1.982,1.983
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Misc In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Misc Modified Files: NEWS Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: NEWS =================================================================== RCS file: /cvsroot/python/python/dist/src/Misc/NEWS,v retrieving revision 1.982 retrieving revision 1.983 diff -C2 -d -r1.982 -r1.983 *** NEWS 2 Jun 2004 09:57:55 -0000 1.982 --- NEWS 2 Jun 2004 16:49:09 -0000 1.983 *************** *** 13,16 **** --- 13,19 ---- ----------------- + - Unicode type got two new methods; iswide() and width(). They + manipulate east asian width information as of Unicode TR11. + - Improved the tuple hashing algorithm to give fewer collisions in common cases. Fixes bug #942952.

1 0

python/dist/src/Lib/test test_unicode.py, 1.87, 1.88
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Lib/test In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Lib/test Modified Files: test_unicode.py Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: test_unicode.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/test/test_unicode.py,v retrieving revision 1.87 retrieving revision 1.88 diff -C2 -d -r1.87 -r1.88 *** test_unicode.py 5 Feb 2004 17:36:00 -0000 1.87 --- test_unicode.py 2 Jun 2004 16:49:09 -0000 1.88 *************** *** 292,295 **** --- 292,315 ---- self.assertRaises(TypeError, u"abc".isnumeric, 42) + def test_iswide(self): + self.checkequalnofix(False, u'', 'iswide') + self.checkequalnofix(False, u'\x1f', 'iswide') # Neutral + self.checkequalnofix(False, u'\x20', 'iswide') # Narrow + self.checkequalnofix(True, u'\u2329', 'iswide') # Wide + self.checkequalnofix(False, u'\uff64', 'iswide') # Half + self.checkequalnofix(True, u'\u3000', 'iswide') # Full + self.checkequalnofix(False, u'\u2460', 'iswide') # Ambiguous + self.checkequalnofix(True, u'\ud55c\uae00', 'iswide') + self.checkequalnofix(False, u'\ud55c\u2606\uae00', 'iswide') + + def test_wide(self): + self.assertEqual(u''.width(), 0) + self.assertEqual(u'abcd'.width(), 4) + self.assertEqual(u'\u0187\u01c9'.width(), 2) + self.assertEqual(u'\u2460\u2329'.width(), 3) + self.assertEqual(u'\u2329\u2460'.width(), 3) + self.assertEqual(u'\ud55c\uae00'.width(), 4) + self.assertEqual(u'\ud55c\u2606\uae00'.width(), 5) + def test_contains(self): # Testing Unicode contains method

1 0

python/dist/src/Doc/lib libstdtypes.tex, 1.153, 1.154
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Doc/lib In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Doc/lib Modified Files: libstdtypes.tex Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: libstdtypes.tex =================================================================== RCS file: /cvsroot/python/python/dist/src/Doc/lib/libstdtypes.tex,v retrieving revision 1.153 retrieving revision 1.154 diff -C2 -d -r1.153 -r1.154 *** libstdtypes.tex 12 May 2004 02:48:29 -0000 1.153 --- libstdtypes.tex 2 Jun 2004 16:49:08 -0000 1.154 *************** *** 643,646 **** --- 643,652 ---- \end{methoddesc} + \begin{methoddesc}[string]{iswide}{} + Return true if all characters in the string are wide or full width and + there is at least one wide or full width character, false otherwise. + This method is supported by unicode type only. + \end{methoddesc} + \begin{methoddesc}[string]{join}{seq} Return a string which is the concatenation of the strings in the *************** *** 775,778 **** --- 781,789 ---- \end{methoddesc} + \begin{methoddesc}[string]{width}{} + Return length of fixed-width representation of the string. This method + is supported by unicode type only. + \end{methoddesc} + \begin{methoddesc}[string]{zfill}{width} Return the numeric string left filled with zeros in a string

1 0

python/dist/src/Include unicodeobject.h, 2.42, 2.43
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Include In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Include Modified Files: unicodeobject.h Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: unicodeobject.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/unicodeobject.h,v retrieving revision 2.42 retrieving revision 2.43 diff -C2 -d -r2.42 -r2.43 *** unicodeobject.h 15 Dec 2003 18:49:19 -0000 2.42 --- unicodeobject.h 2 Jun 2004 16:49:08 -0000 2.43 *************** *** 181,184 **** --- 181,185 ---- # define PyUnicode_GetMax PyUnicodeUCS2_GetMax # define PyUnicode_GetSize PyUnicodeUCS2_GetSize + # define PyUnicode_GetWidth PyUnicodeUCS2_GetWidth # define PyUnicode_Join PyUnicodeUCS2_Join # define PyUnicode_Replace PyUnicodeUCS2_Replace *************** *** 200,203 **** --- 201,205 ---- # define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase # define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric + # define _PyUnicode_IsWide _PyUnicodeUCS2_IsWide # define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase # define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase *************** *** 253,256 **** --- 255,259 ---- # define PyUnicode_GetMax PyUnicodeUCS4_GetMax # define PyUnicode_GetSize PyUnicodeUCS4_GetSize + # define PyUnicode_GetWidth PyUnicodeUCS4_GetWidth # define PyUnicode_Join PyUnicodeUCS4_Join # define PyUnicode_Replace PyUnicodeUCS4_Replace *************** *** 271,274 **** --- 274,278 ---- # define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase # define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric + # define _PyUnicode_IsWide _PyUnicodeUCS4_IsWide # define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase # define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase *************** *** 316,319 **** --- 320,325 ---- #define Py_UNICODE_ISALPHA(ch) iswalpha(ch) + #define Py_UNICODE_ISWIDE(ch) _PyUnicode_IsWide(ch) + #else *************** *** 339,342 **** --- 345,350 ---- #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch) + #define Py_UNICODE_ISWIDE(ch) _PyUnicode_IsWide(ch) + #endif *************** *** 431,434 **** --- 439,448 ---- ); + /* Get the fixed-width representation length of the Unicode object */ + + PyAPI_FUNC(int) PyUnicode_GetWidth( + PyObject *unicode /* Unicode object */ + ); + /* Get the maximum ordinal for a Unicode character. */ PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void); *************** *** 1152,1155 **** --- 1166,1173 ---- ); + PyAPI_FUNC(int) _PyUnicode_IsWide( + Py_UNICODE ch /* Unicode character */ + ); + #ifdef __cplusplus }

1 0

python/dist/src/Doc/api concrete.tex,1.41,1.42
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Doc/api In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Doc/api Modified Files: concrete.tex Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: concrete.tex =================================================================== RCS file: /cvsroot/python/python/dist/src/Doc/api/concrete.tex,v retrieving revision 1.41 retrieving revision 1.42 diff -C2 -d -r1.41 -r1.42 *** concrete.tex 2 Jun 2004 12:45:27 -0000 1.41 --- concrete.tex 2 Jun 2004 16:49:07 -0000 1.42 *************** *** 851,854 **** --- 851,859 ---- \end{cfuncdesc} + \begin{cfuncdesc}{int}{Py_UNICODE_ISWIDE}{Py_UNICODE ch} + Returns 1/0 depending on whether \var{ch} is a wide or full-width + character. + \end{cfuncdesc} + These APIs can be used for fast direct character conversions: *************** *** 909,912 **** --- 914,921 ---- \end{cfuncdesc} + \begin{cfuncdesc}{int}{PyUnicode_GetWidth}{PyObject *unicode} + Return the fixed-width representation length of the Unicode object. + \end{cfuncdesc} + \begin{cfuncdesc}{PyObject*}{PyUnicode_FromEncodedObject}{PyObject *obj, const char *encoding,

1 0

python/dist/src/Objects unicodectype.c, 2.14, 2.15 unicodeobject.c, 2.211, 2.212 unicodetype_db.h, 1.7, 1.8
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Objects In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Objects Modified Files: unicodectype.c unicodeobject.c unicodetype_db.h Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: unicodectype.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Objects/unicodectype.c,v retrieving revision 2.14 retrieving revision 2.15 diff -C2 -d -r2.14 -r2.15 *** unicodectype.c 29 Dec 2003 01:36:01 -0000 2.14 --- unicodectype.c 2 Jun 2004 16:49:16 -0000 2.15 *************** *** 20,26 **** #define TITLE_MASK 0x40 #define UPPER_MASK 0x80 typedef struct { - const unsigned short flags; const Py_UNICODE upper; const Py_UNICODE lower; --- 20,26 ---- #define TITLE_MASK 0x40 #define UPPER_MASK 0x80 + #define WIDE_MASK 0x100 typedef struct { const Py_UNICODE upper; const Py_UNICODE lower; *************** *** 28,31 **** --- 28,32 ---- const unsigned char decimal; const unsigned char digit; + const unsigned short flags; } _PyUnicode_TypeRecord; *************** *** 323,326 **** --- 324,336 ---- } + /* Returns 1 for Unicode characters having Full or Wide width, 0 otherwise */ + + int _PyUnicode_IsWide(Py_UNICODE ch) + { + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + return (ctype->flags & WIDE_MASK) != 0; + } + #ifndef WANT_WCTYPE_FUNCTIONS Index: unicodeobject.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Objects/unicodeobject.c,v retrieving revision 2.211 retrieving revision 2.212 diff -C2 -d -r2.211 -r2.212 *** unicodeobject.c 6 Apr 2004 07:24:51 -0000 2.211 --- unicodeobject.c 2 Jun 2004 16:49:16 -0000 2.212 *************** *** 656,659 **** --- 656,680 ---- } + int PyUnicode_GetWidth(PyObject *unicode) + { + const Py_UNICODE *p, *e; + int width; + + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return -1; + } + + p = PyUnicode_AS_UNICODE(unicode); + e = p + PyUnicode_GET_SIZE(unicode); + for (width = 0; p < e; p++) + if (Py_UNICODE_ISWIDE(*p)) + width += 2; + else + width++; + + return width; + } + const char *PyUnicode_GetDefaultEncoding(void) { *************** *** 5317,5320 **** --- 5338,5370 ---- } + PyDoc_STRVAR(iswide__doc__, + "S.iswide() -> bool\n\ + \n\ + Return True if all characters in S are wide width\n\ + and there is at least one character in S, False otherwise."); + + static PyObject* + unicode_iswide(PyUnicodeObject *self) + { + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; + + /* Shortcut for single character strings */ + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISWIDE(*p)) + Py_RETURN_TRUE; + + /* Special case for empty strings */ + if (PyString_GET_SIZE(self) == 0) + Py_RETURN_FALSE; + + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISWIDE(*p)) + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; + } + PyDoc_STRVAR(join__doc__, "S.join(sequence) -> unicode\n\ *************** *** 5336,5340 **** PyDoc_STRVAR(ljust__doc__, ! "S.ljust(width[, fillchar]) -> unicode\n\ \n\ Return S left justified in a Unicode string of length width. Padding is\n\ --- 5386,5390 ---- PyDoc_STRVAR(ljust__doc__, ! "S.ljust(width[, fillchar]) -> int\n\ \n\ Return S left justified in a Unicode string of length width. Padding is\n\ *************** *** 5928,5931 **** --- 5978,5996 ---- } + PyDoc_STRVAR(width__doc__, + "S.width() -> unicode\n\ + \n\ + Return a fixed-width representation length of S."); + + static PyObject* + unicode_width(PyObject *self) + { + int width = PyUnicode_GetWidth(self); + if (width == -1) + return NULL; + else + return PyInt_FromLong((long)width); + } + PyDoc_STRVAR(zfill__doc__, "S.zfill(width) -> unicode\n\ *************** *** 6091,6094 **** --- 6156,6161 ---- {"isalpha", (PyCFunction) unicode_isalpha, METH_NOARGS, isalpha__doc__}, {"isalnum", (PyCFunction) unicode_isalnum, METH_NOARGS, isalnum__doc__}, + {"iswide", (PyCFunction) unicode_iswide, METH_NOARGS, iswide__doc__}, + {"width", (PyCFunction) unicode_width, METH_NOARGS, width__doc__}, {"zfill", (PyCFunction) unicode_zfill, METH_VARARGS, zfill__doc__}, #if 0 Index: unicodetype_db.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Objects/unicodetype_db.h,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** unicodetype_db.h 25 Nov 2002 09:13:36 -0000 1.7 --- unicodetype_db.h 2 Jun 2004 16:49:16 -0000 1.8 *************** *** 1,3 **** ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.2 */ /* a list of unique character type descriptors */ --- 1,3 ---- ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.3 */ /* a list of unique character type descriptors */ *************** *** 5,131 **** {0, 0, 0, 0, 0, 0}, [...1225 lines suppressed...] 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ! 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, *************** *** 1086,1090 **** 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ! 1, 1, 1, 1, 1, 1, 0, 0, }; --- 1147,1151 ---- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ! 1, 1, 1, 1, 1, 0, 0, };

1 0

python/dist/src/Modules unicodedata_db.h, 1.9, 1.10 unicodename_db.h, 1.6, 1.7
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Modules In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Modules Modified Files: unicodedata_db.h unicodename_db.h Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: unicodedata_db.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/unicodedata_db.h,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** unicodedata_db.h 25 Nov 2002 09:13:35 -0000 1.9 --- unicodedata_db.h 2 Jun 2004 16:49:12 -0000 1.10 *************** *** 1,3 **** ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.2 */ #define UNIDATA_VERSION "3.2.0" --- 1,3 ---- ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.3 */ #define UNIDATA_VERSION "3.2.0" Index: unicodename_db.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/unicodename_db.h,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** unicodename_db.h 25 Nov 2002 09:13:36 -0000 1.6 --- unicodename_db.h 2 Jun 2004 16:49:13 -0000 1.7 *************** *** 1,3 **** ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.2 */ #define NAME_MAXLEN 256 --- 1,3 ---- ! /* this file was generated by Tools/unicode/makeunicodedata.py 2.3 */ #define NAME_MAXLEN 256

1 0

python/dist/src/Tools/unicode makeunicodedata.py, 1.17, 1.18
by perky＠users.sourceforge.net June 2, 2004

June 2, 2004

Update of /cvsroot/python/python/dist/src/Tools/unicode In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv32555/Tools/unicode Modified Files: makeunicodedata.py Log Message: - SF #962502: Add two more methods for unicode type; width() and iswide() for east asian width manipulation. (Inspired by David Goodger, Reviewed by Martin v. Loewis) - Move _PyUnicode_TypeRecord.flags to the end of the struct so that no padding is added for UCS-4 builds. (Suggested by Martin v. Loewis) Index: makeunicodedata.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Tools/unicode/makeunicodedata.py,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** makeunicodedata.py 25 Nov 2002 09:13:37 -0000 1.17 --- makeunicodedata.py 2 Jun 2004 16:49:17 -0000 1.18 *************** *** 19,22 **** --- 19,23 ---- # 2002-11-24 mvl expand all ranges, sort names version-independently # 2002-11-25 mvl add UNIDATA_VERSION + # 2004-05-29 perky add east asian width information # # written by Fredrik Lundh (fredrik(a)pythonware.com) *************** *** 26,30 **** SCRIPT = sys.argv[0] ! VERSION = "2.2" # The Unicode Database --- 27,31 ---- SCRIPT = sys.argv[0] ! VERSION = "2.3" # The Unicode Database *************** *** 32,35 **** --- 33,37 ---- UNICODE_DATA = "UnicodeData.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions.txt" + EASTASIAN_WIDTH = "EastAsianWidth.txt" CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd", *************** *** 51,54 **** --- 53,57 ---- TITLE_MASK = 0x40 UPPER_MASK = 0x80 + WIDE_MASK = 0x100 def maketables(trace=0): *************** *** 56,60 **** print "--- Reading", UNICODE_DATA, "..." ! unicode = UnicodeData(UNICODE_DATA, COMPOSITION_EXCLUSIONS) print len(filter(None, unicode.table)), "characters" --- 59,64 ---- print "--- Reading", UNICODE_DATA, "..." ! unicode = UnicodeData(UNICODE_DATA, COMPOSITION_EXCLUSIONS, ! EASTASIAN_WIDTH) print len(filter(None, unicode.table)), "characters" *************** *** 331,336 **** flags |= DIGIT_MASK digit = int(record[7]) item = ( ! flags, upper, lower, title, decimal, digit ) # add entry to index and item tables --- 335,342 ---- flags |= DIGIT_MASK digit = int(record[7]) + if record[15] in ('W', 'F'): # Wide or Full width + flags |= WIDE_MASK item = ( ! upper, lower, title, decimal, digit, flags ) # add entry to index and item tables *************** *** 539,543 **** class UnicodeData: ! def __init__(self, filename, exclusions, expand=1): file = open(filename) table = [None] * 0x110000 --- 545,549 ---- class UnicodeData: ! def __init__(self, filename, exclusions, eastasianwidth, expand=1): file = open(filename) table = [None] * 0x110000 *************** *** 582,585 **** --- 588,610 ---- self.exclusions[char] = 1 + widths = [None] * 0x110000 + for s in open(eastasianwidth): + s = s.strip() + if not s: + continue + if s[0] == '#': + continue + s = s.split()[0].split(';') + if '..' in s[0]: + first, last = [int(c, 16) for c in s[0].split('..')] + chars = range(first, last+1) + else: + chars = [int(s[0], 16)] + for char in chars: + widths[char] = s[1] + for i in range(0, 0x110000): + if table[i] is not None: + table[i].append(widths[i]) + def uselatin1(self): # restrict character range to ISO Latin 1

1 0