Mailman 3 September 2011 - Python-checkins

cpython: posix module catches PyUnicode_AsUnicode() failure
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/f265b20ac9c9 changeset: 72542:f265b20ac9c9 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Fri Sep 30 01:44:27 2011 +0200 summary: posix module catches PyUnicode_AsUnicode() failure * Replace PyUnicode_AS_UNICODE by PyUnicode_AsUnicode, PyUnicode_AS_UNICODE is no more a real macro * Replace Py_UNICODE by wchar_t in code specific to Windows files: Modules/posixmodule.c | 279 ++++++++++++++++++----------- 1 files changed, 175 insertions(+), 104 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -697,7 +697,7 @@ } static PyObject * -win32_error_unicode(char* function, Py_UNICODE* filename) +win32_error_unicode(char* function, wchar_t* filename) { /* XXX - see win32_error for comments on 'function' */ errno = GetLastError(); @@ -707,6 +707,20 @@ return PyErr_SetFromWindowsErr(errno); } +static PyObject * +win32_error_object(char* function, PyObject* filename) +{ + /* XXX - see win32_error for comments on 'function' */ + errno = GetLastError(); + if (filename) + return PyErr_SetExcFromWindowsErrWithFilenameObject( + PyExc_WindowsError, + errno, + filename); + else + return PyErr_SetFromWindowsErr(errno); +} + static int convert_to_unicode(PyObject **param) { @@ -881,17 +895,21 @@ char *ansi; BOOL result; - if (!PyArg_ParseTuple(args, wformat, &uni)) - PyErr_Clear(); - else { + if (PyArg_ParseTuple(args, wformat, &uni)) + { + wchar_t *wstr = PyUnicode_AsUnicode(uni); + if (wstr == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - result = funcW(PyUnicode_AsUnicode(uni)); + result = funcW(wstr); Py_END_ALLOW_THREADS if (!result) - return win32_error_unicode(func, PyUnicode_AsUnicode(uni)); + return win32_error_object(func, uni); Py_INCREF(Py_None); return Py_None; } + PyErr_Clear(); + if (!PyArg_ParseTuple(args, format, &ansi)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -1801,7 +1819,7 @@ int (*statfunc)(const char *, STRUCT_STAT *), #endif char *wformat, - int (*wstatfunc)(const Py_UNICODE *, STRUCT_STAT *)) + int (*wstatfunc)(const wchar_t *, STRUCT_STAT *)) { STRUCT_STAT st; PyObject *opath; @@ -1810,18 +1828,18 @@ PyObject *result; #ifdef MS_WINDOWS - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, wformat, &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE result OK without - thread lock as it is a simple dereference. */ res = wstatfunc(wpath, &st); Py_END_ALLOW_THREADS if (res != 0) - return win32_error_unicode("stat", wpath); + return win32_error_object("stat", po); return _pystat_fromstructstat(&st); } /* Drop the argument parsing error as narrow strings @@ -1870,12 +1888,13 @@ #ifdef MS_WINDOWS DWORD attr; - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui:access", &po, &mode)) { + wchar_t* wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ - attr = GetFileAttributesW(PyUnicode_AS_UNICODE(po)); + attr = GetFileAttributesW(wpath); Py_END_ALLOW_THREADS goto finish; } @@ -2025,23 +2044,25 @@ int res; #ifdef MS_WINDOWS DWORD attr; - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui|:chmod", &po, &i)) { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - attr = GetFileAttributesW(PyUnicode_AS_UNICODE(po)); + attr = GetFileAttributesW(wpath); if (attr != 0xFFFFFFFF) { if (i & _S_IWRITE) attr &= ~FILE_ATTRIBUTE_READONLY; else attr |= FILE_ATTRIBUTE_READONLY; - res = SetFileAttributesW(PyUnicode_AS_UNICODE(po), attr); + res = SetFileAttributesW(wpath, attr); } else res = 0; Py_END_ALLOW_THREADS if (!res) - return win32_error_unicode("chmod", - PyUnicode_AS_UNICODE(po)); + return win32_error_object("chmod", po); Py_INCREF(Py_None); return Py_None; } @@ -2429,12 +2450,20 @@ PyObject *osrc, *odst; char *src, *dst; BOOL rslt; - - PyUnicodeObject *usrc, *udst; - if (PyArg_ParseTuple(args, "UU:link", &usrc, &udst)) { + PyObject *usrc, *udst; + + if (PyArg_ParseTuple(args, "UU:link", &usrc, &udst)) + { + wchar_t *wsrc, *wdst; + wsrc = PyUnicode_AsUnicode(usrc); + if (wsrc == NULL) + return NULL; + wdst = PyUnicode_AsUnicode(udst); + if (wdst == NULL) + return NULL; + Py_BEGIN_ALLOW_THREADS - rslt = CreateHardLinkW(PyUnicode_AS_UNICODE(udst), - PyUnicode_AS_UNICODE(usrc), NULL); + rslt = CreateHardLinkW(wdst, wsrc, NULL); Py_END_ALLOW_THREADS if (rslt == 0) @@ -2495,13 +2524,15 @@ PyObject *po = NULL; if (PyArg_ParseTuple(args, "|U:listdir", &po)) { WIN32_FIND_DATAW wFileData; - Py_UNICODE *wnamebuf, *po_wchars; + wchar_t *wnamebuf, *po_wchars; if (po == NULL) { /* Default arg: "." */ po_wchars = L"."; len = 1; } else { - po_wchars = PyUnicode_AS_UNICODE(po); + po_wchars = PyUnicode_AsUnicode(po); + if (po_wchars == NULL) + return NULL; len = PyUnicode_GET_SIZE(po); } /* Overallocate for \\*.*\0 */ @@ -2512,7 +2543,7 @@ } wcscpy(wnamebuf, po_wchars); if (len > 0) { - Py_UNICODE wch = wnamebuf[len-1]; + wchar_t wch = wnamebuf[len-1]; if (wch != L'/' && wch != L'\\' && wch != L':') wnamebuf[len++] = L'\\'; wcscpy(wnamebuf + len, L"*.*"); @@ -2887,18 +2918,24 @@ char outbuf[MAX_PATH*2]; char *temp; #ifdef MS_WINDOWS - PyUnicodeObject *po; - if (PyArg_ParseTuple(args, "U|:_getfullpathname", &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); - Py_UNICODE woutbuf[MAX_PATH*2], *woutbufp = woutbuf; - Py_UNICODE *wtemp; + PyObject *po; + + if (PyArg_ParseTuple(args, "U|:_getfullpathname", &po)) + { + wchar_t *wpath; + wchar_t woutbuf[MAX_PATH*2], *woutbufp = woutbuf; + wchar_t *wtemp; DWORD result; PyObject *v; + + wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; result = GetFullPathNameW(wpath, Py_ARRAY_LENGTH(woutbuf), woutbuf, &wtemp); if (result > Py_ARRAY_LENGTH(woutbuf)) { - woutbufp = malloc(result * sizeof(Py_UNICODE)); + woutbufp = malloc(result * sizeof(wchar_t)); if (!woutbufp) return PyErr_NoMemory(); result = GetFullPathNameW(wpath, result, woutbufp, &wtemp); @@ -2906,7 +2943,7 @@ if (result) v = PyUnicode_FromUnicode(woutbufp, wcslen(woutbufp)); else - v = win32_error_unicode("GetFullPathNameW", wpath); + v = win32_error_object("GetFullPathNameW", po); if (woutbufp != woutbuf) free(woutbufp); return v; @@ -2914,8 +2951,8 @@ /* Drop the argument parsing error as narrow strings are also valid. */ PyErr_Clear(); - -#endif +#endif + if (!PyArg_ParseTuple (args, "O&:_getfullpathname", PyUnicode_FSConverter, &opath)) return NULL; @@ -2944,12 +2981,14 @@ int buf_size; wchar_t *target_path; int result_length; - PyObject *result; + PyObject *po, *result; wchar_t *path; - if (!PyArg_ParseTuple(args, "u|:_getfinalpathname", &path)) { - return NULL; - } + if (!PyArg_ParseTuple(args, "U|:_getfinalpathname", &po)) + return NULL; + path = PyUnicode_AsUnicode(po); + if (path == NULL) + return NULL; if(!check_GetFinalPathNameByHandle()) { /* If the OS doesn't have GetFinalPathNameByHandle, return a @@ -2968,18 +3007,15 @@ FILE_FLAG_BACKUP_SEMANTICS, NULL); - if(hFile == INVALID_HANDLE_VALUE) { - return win32_error_unicode("GetFinalPathNamyByHandle", path); - return PyErr_Format(PyExc_RuntimeError, - "Could not get a handle to file."); - } + if(hFile == INVALID_HANDLE_VALUE) + return win32_error_object("CreateFileW", po); /* We have a good handle to the target, use it to determine the target path name. */ buf_size = Py_GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); if(!buf_size) - return win32_error_unicode("GetFinalPathNameByHandle", path); + return win32_error_object("GetFinalPathNameByHandle", po); target_path = (wchar_t *)malloc((buf_size+1)*sizeof(wchar_t)); if(!target_path) @@ -2988,10 +3024,10 @@ result_length = Py_GetFinalPathNameByHandleW(hFile, target_path, buf_size, VOLUME_NAME_DOS); if(!result_length) - return win32_error_unicode("GetFinalPathNamyByHandle", path); + return win32_error_object("GetFinalPathNamyByHandle", po); if(!CloseHandle(hFile)) - return win32_error_unicode("GetFinalPathNameByHandle", path); + return win32_error_object("CloseHandle", po); target_path[result_length] = 0; result = PyUnicode_FromUnicode(target_path, result_length); @@ -3033,11 +3069,13 @@ { PyObject *opath; char *path; - PyUnicodeObject *po; + PyObject *po; DWORD attributes; if (PyArg_ParseTuple(args, "U|:_isdir", &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; attributes = GetFileAttributesW(wpath); if (attributes == INVALID_FILE_ATTRIBUTES) @@ -3078,15 +3116,18 @@ int mode = 0777; #ifdef MS_WINDOWS - PyUnicodeObject *po; - if (PyArg_ParseTuple(args, "U|i:mkdir", &po, &mode)) { + PyObject *po; + if (PyArg_ParseTuple(args, "U|i:mkdir", &po, &mode)) + { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; + Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ - res = CreateDirectoryW(PyUnicode_AS_UNICODE(po), NULL); + res = CreateDirectoryW(wpath, NULL); Py_END_ALLOW_THREADS if (!res) - return win32_error_unicode("mkdir", PyUnicode_AS_UNICODE(po)); + return win32_error_object("mkdir", po); Py_INCREF(Py_None); return Py_None; } @@ -3098,8 +3139,6 @@ return NULL; path = PyBytes_AsString(opath); Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ res = CreateDirectoryA(path, NULL); Py_END_ALLOW_THREADS if (!res) { @@ -3225,6 +3264,7 @@ { #ifdef MS_WINDOWS PyObject *o1, *o2; + wchar_t *w1, *w2; char *p1, *p2; BOOL result; if (!PyArg_ParseTuple(args, "OO:rename", &o1, &o2)) @@ -3235,9 +3275,14 @@ Py_DECREF(o1); goto error; } - Py_BEGIN_ALLOW_THREADS - result = MoveFileW(PyUnicode_AsUnicode(o1), - PyUnicode_AsUnicode(o2)); + w1 = PyUnicode_AsUnicode(o1); + if (w1 == NULL) + goto error; + w2 = PyUnicode_AsUnicode(o2); + if (w2 == NULL) + goto error; + Py_BEGIN_ALLOW_THREADS + result = MoveFileW(w1, w2); Py_END_ALLOW_THREADS Py_DECREF(o1); Py_DECREF(o2); @@ -3497,7 +3542,7 @@ { #ifdef MS_WINDOWS PyObject *arg; - PyUnicodeObject *obwpath; + PyObject *obwpath; wchar_t *wpath = NULL; PyObject *oapath; char *apath; @@ -3508,23 +3553,26 @@ PyObject *result = NULL; if (PyArg_ParseTuple(args, "UO|:utime", &obwpath, &arg)) { - wpath = PyUnicode_AS_UNICODE(obwpath); + wpath = PyUnicode_AsUnicode(obwpath); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW(wpath, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) - return win32_error_unicode("utime", wpath); - } else + return win32_error_object("utime", obwpath); + } + else { /* Drop the argument parsing error as narrow strings are also valid. */ PyErr_Clear(); - if (!wpath) { if (!PyArg_ParseTuple(args, "O&O:utime", PyUnicode_FSConverter, &oapath, &arg)) return NULL; + apath = PyBytes_AsString(oapath); Py_BEGIN_ALLOW_THREADS hFile = CreateFileA(apath, FILE_WRITE_ATTRIBUTES, 0, @@ -6372,7 +6420,7 @@ wchar_t *path; DWORD n_bytes_returned; DWORD io_result; - PyObject *result; + PyObject *po, *result; HANDLE reparse_point_handle; char target_buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; @@ -6380,8 +6428,11 @@ wchar_t *print_name; if (!PyArg_ParseTuple(args, - "u:readlink", - &path)) + "U:readlink", + &po)) + return NULL; + path = PyUnicode_AsUnicode(po); + if (path == NULL) return NULL; /* First get a handle to the reparse point */ @@ -6397,9 +6448,7 @@ Py_END_ALLOW_THREADS if (reparse_point_handle==INVALID_HANDLE_VALUE) - { - return win32_error_unicode("readlink", path); - } + return win32_error_object("readlink", po); Py_BEGIN_ALLOW_THREADS /* New call DeviceIoControl to read the reparse point */ @@ -6415,9 +6464,7 @@ Py_END_ALLOW_THREADS if (io_result==0) - { - return win32_error_unicode("readlink", path); - } + return win32_error_object("readlink", po); if (rdb->ReparseTag != IO_REPARSE_TAG_SYMLINK) { @@ -6468,6 +6515,7 @@ { static char *kwlist[] = {"src", "dest", "target_is_directory", NULL}; PyObject *src, *dest; + wchar_t *wsrc, *wdest; int target_is_directory = 0; DWORD res; WIN32_FILE_ATTRIBUTE_DATA src_info; @@ -6485,16 +6533,24 @@ if (win32_can_symlink == 0) return PyErr_Format(PyExc_OSError, "symbolic link privilege not held"); - if (!convert_to_unicode(&src)) { return NULL; } + if (!convert_to_unicode(&src)) + return NULL; if (!convert_to_unicode(&dest)) { Py_DECREF(src); return NULL; } + wsrc = PyUnicode_AsUnicode(src); + if (wsrc == NULL) + goto error; + wdest = PyUnicode_AsUnicode(dest); + if (wsrc == NULL) + goto error; + /* if src is a directory, ensure target_is_directory==1 */ if( GetFileAttributesExW( - PyUnicode_AsUnicode(src), GetFileExInfoStandard, &src_info + wsrc, GetFileExInfoStandard, &src_info )) { target_is_directory = target_is_directory || @@ -6502,20 +6558,21 @@ } Py_BEGIN_ALLOW_THREADS - res = Py_CreateSymbolicLinkW( - PyUnicode_AsUnicode(dest), - PyUnicode_AsUnicode(src), - target_is_directory); - Py_END_ALLOW_THREADS + res = Py_CreateSymbolicLinkW(wdest, wsrc, target_is_directory); + Py_END_ALLOW_THREADS + Py_DECREF(src); Py_DECREF(dest); if (!res) - { - return win32_error_unicode("symlink", PyUnicode_AsUnicode(src)); - } + return win32_error_object("symlink", src); Py_INCREF(Py_None); return Py_None; + +error: + Py_DECREF(src); + Py_DECREF(dest); + return NULL; } #endif /* defined(HAVE_SYMLINK) && defined(MS_WINDOWS) */ @@ -6710,12 +6767,14 @@ int fd; #ifdef MS_WINDOWS - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui|i:open", &po, &flag, &mode)) { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; + Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread - lock as it is a simple dereference. */ - fd = _wopen(PyUnicode_AS_UNICODE(po), flag, mode); + fd = _wopen(wpath, flag, mode); Py_END_ALLOW_THREADS if (fd < 0) return posix_error(); @@ -7717,6 +7776,8 @@ } #ifdef MS_WINDOWS newenv = PyUnicode_AsUnicode(newstr); + if (newenv == NULL) + goto error; _snwprintf(newenv, len, L"%s=%s", s1, s2); if (_wputenv(newenv)) { posix_error(); @@ -9178,9 +9239,10 @@ PyObject *ofilepath; char *filepath; char *operation = NULL; + wchar_t *wpath, *woperation; HINSTANCE rc; - PyObject *unipath, *woperation = NULL; + PyObject *unipath, *uoperation = NULL; if (!PyArg_ParseTuple(args, "U|s:startfile", &unipath, &operation)) { PyErr_Clear(); @@ -9188,26 +9250,35 @@ } if (operation) { - woperation = PyUnicode_DecodeASCII(operation, + uoperation = PyUnicode_DecodeASCII(operation, strlen(operation), NULL); - if (!woperation) { + if (!uoperation) { PyErr_Clear(); operation = NULL; goto normal; } } - Py_BEGIN_ALLOW_THREADS - rc = ShellExecuteW((HWND)0, woperation ? PyUnicode_AS_UNICODE(woperation) : 0, - PyUnicode_AS_UNICODE(unipath), - NULL, NULL, SW_SHOWNORMAL); - Py_END_ALLOW_THREADS - - Py_XDECREF(woperation); + wpath = PyUnicode_AsUnicode(unipath); + if (wpath == NULL) + goto normal; + if (uoperation) { + woperation = PyUnicode_AsUnicode(uoperation); + if (woperation == NULL) + goto normal; + } + else + woperation = NULL; + + Py_BEGIN_ALLOW_THREADS + rc = ShellExecuteW((HWND)0, woperation, wpath, + NULL, NULL, SW_SHOWNORMAL); + Py_END_ALLOW_THREADS + + Py_XDECREF(uoperation); if (rc <= (HINSTANCE)32) { - PyObject *errval = win32_error_unicode("startfile", - PyUnicode_AS_UNICODE(unipath)); - return errval; + win32_error_object("startfile", unipath); + return NULL; } Py_INCREF(Py_None); return Py_None; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: array module stores the typecode in a char, instead of Py_UNICODE
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/7042a83f37e6 changeset: 72540:7042a83f37e6 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Fri Sep 30 00:03:59 2011 +0200 summary: array module stores the typecode in a char, instead of Py_UNICODE files: Modules/arraymodule.c | 35 ++++++++++++++---------------- 1 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -22,7 +22,7 @@ * functions aren't visible yet. */ struct arraydescr { - Py_UNICODE typecode; + char typecode; int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); @@ -1510,7 +1510,7 @@ { Py_UNICODE *ustr; Py_ssize_t n; - Py_UNICODE typecode; + char typecode; if (!PyArg_ParseTuple(args, "u#:fromunicode", &ustr, &n)) return NULL; @@ -1545,7 +1545,7 @@ static PyObject * array_tounicode(arrayobject *self, PyObject *unused) { - Py_UNICODE typecode; + char typecode; typecode = self->ob_descr->typecode; if ((typecode != 'u')) { PyErr_SetString(PyExc_ValueError, @@ -1642,7 +1642,7 @@ * be found. */ static enum machine_format_code -typecode_to_mformat_code(int typecode) +typecode_to_mformat_code(char typecode) { #ifdef WORDS_BIGENDIAN const int is_big_endian = 1; @@ -1721,7 +1721,7 @@ intsize = sizeof(PY_LONG_LONG); is_signed = 0; break; -#endif +#endif default: return UNKNOWN_FORMAT; } @@ -1752,7 +1752,7 @@ * NULL is returned to indicate a failure. */ static PyObject * -make_array(PyTypeObject *arraytype, Py_UNICODE typecode, PyObject *items) +make_array(PyTypeObject *arraytype, char typecode, PyObject *items) { PyObject *new_args; PyObject *array_obj; @@ -1761,7 +1761,7 @@ assert(arraytype != NULL); assert(items != NULL); - typecode_obj = PyUnicode_FromUnicode(&typecode, 1); + typecode_obj = PyUnicode_FromOrdinal(typecode); if (typecode_obj == NULL) return NULL; @@ -1791,17 +1791,14 @@ PyObject *items; PyObject *converted_items; PyObject *result; - int typecode_int; - Py_UNICODE typecode; + int typecode; enum machine_format_code mformat_code; struct arraydescr *descr; if (!PyArg_ParseTuple(args, "OCiO:array._array_reconstructor", - &arraytype, &typecode_int, &mformat_code, &items)) + &arraytype, &typecode, &mformat_code, &items)) return NULL; - typecode = (Py_UNICODE)typecode_int; - if (!PyType_Check(arraytype)) { PyErr_Format(PyExc_TypeError, "first argument must a type object, not %.200s", @@ -1815,7 +1812,7 @@ return NULL; } for (descr = descriptors; descr->typecode != '\0'; descr++) { - if (descr->typecode == typecode) + if ((int)descr->typecode == typecode) break; } if (descr->typecode == '\0') { @@ -1837,9 +1834,9 @@ } /* Fast path: No decoding has to be done. */ - if (mformat_code == typecode_to_mformat_code(typecode) || + if (mformat_code == typecode_to_mformat_code((char)typecode) || mformat_code == UNKNOWN_FORMAT) { - return make_array(arraytype, typecode, items); + return make_array(arraytype, (char)typecode, items); } /* Slow path: Decode the byte string according to the given machine @@ -1985,7 +1982,7 @@ return NULL; } - result = make_array(arraytype, typecode, converted_items); + result = make_array(arraytype, (char)typecode, converted_items); Py_DECREF(converted_items); return result; } @@ -2074,8 +2071,8 @@ static PyObject * array_get_typecode(arrayobject *a, void *closure) { - Py_UNICODE tc = a->ob_descr->typecode; - return PyUnicode_FromUnicode(&tc, 1); + char typecode = a->ob_descr->typecode; + return PyUnicode_FromOrdinal(typecode); } static PyObject * @@ -2147,7 +2144,7 @@ static PyObject * array_repr(arrayobject *a) { - Py_UNICODE typecode; + char typecode; PyObject *s, *v = NULL; Py_ssize_t len; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: array module uses the new Unicode API
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/7fa098f6dc6a changeset: 72541:7fa098f6dc6a user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Fri Sep 30 00:51:10 2011 +0200 summary: array module uses the new Unicode API * Use Py_UCS4* buffer instead of Py_UNICODE* * Use "I" or "L" format, instead of "u" format files: Lib/test/test_array.py | 8 ++- Modules/arraymodule.c | 63 ++++++++++++++--------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -218,10 +218,14 @@ self.assertEqual(bi[1], len(a)) def test_byteswap(self): - a = array.array(self.typecode, self.example) + if self.typecode == 'u': + example = '\U00100100' + else: + example = self.example + a = array.array(self.typecode, example) self.assertRaises(TypeError, a.byteswap, 42) if a.itemsize in (1, 2, 4, 8): - b = array.array(self.typecode, self.example) + b = array.array(self.typecode, example) b.byteswap() if a.itemsize==1: self.assertEqual(a, b) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -174,24 +174,25 @@ static PyObject * u_getitem(arrayobject *ap, Py_ssize_t i) { - return PyUnicode_FromUnicode(&((Py_UNICODE *) ap->ob_item)[i], 1); + return PyUnicode_FromOrdinal(((Py_UCS4 *) ap->ob_item)[i]); } static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - Py_UNICODE *p; - Py_ssize_t len; + PyObject *p; - if (!PyArg_Parse(v, "u#;array item must be unicode character", &p, &len)) + if (!PyArg_Parse(v, "U;array item must be unicode character", &p)) return -1; - if (len != 1) { + if (PyUnicode_READY(p)) + return -1; + if (PyUnicode_GET_LENGTH(p) != 1) { PyErr_SetString(PyExc_TypeError, "array item must be unicode character"); return -1; } if (i >= 0) - ((Py_UNICODE *)ap->ob_item)[i] = p[0]; + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(p, 0); return 0; } @@ -443,6 +444,13 @@ return 0; } +#if SIZEOF_INT == 4 +# define STRUCT_LONG_FORMAT "I" +#elif SIZEOF_LONG == 4 +# define STRUCT_LONG_FORMAT "L" +#else +# error "Unable to get struct format for Py_UCS4" +#endif /* Description of types. * @@ -452,7 +460,7 @@ static struct arraydescr descriptors[] = { {'b', 1, b_getitem, b_setitem, "b", 1, 1}, {'B', 1, BB_getitem, BB_setitem, "B", 1, 0}, - {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u", 0, 0}, + {'u', sizeof(Py_UCS4), u_getitem, u_setitem, STRUCT_LONG_FORMAT, 0, 0}, {'h', sizeof(short), h_getitem, h_setitem, "h", 1, 1}, {'H', sizeof(short), HH_getitem, HH_setitem, "H", 1, 0}, {'i', sizeof(int), i_getitem, i_setitem, "i", 1, 1}, @@ -1508,25 +1516,26 @@ static PyObject * array_fromunicode(arrayobject *self, PyObject *args) { - Py_UNICODE *ustr; + PyObject *ustr; Py_ssize_t n; - char typecode; - if (!PyArg_ParseTuple(args, "u#:fromunicode", &ustr, &n)) + if (!PyArg_ParseTuple(args, "U:fromunicode", &ustr)) return NULL; - typecode = self->ob_descr->typecode; - if ((typecode != 'u')) { + if (self->ob_descr->typecode != 'u') { PyErr_SetString(PyExc_ValueError, "fromunicode() may only be called on " "unicode type arrays"); return NULL; } + if (PyUnicode_READY(ustr)) + return NULL; + n = PyUnicode_GET_LENGTH(ustr); if (n > 0) { Py_ssize_t old_size = Py_SIZE(self); if (array_resize(self, old_size + n) == -1) return NULL; - memcpy(self->ob_item + old_size * sizeof(Py_UNICODE), - ustr, n * sizeof(Py_UNICODE)); + if (!PyUnicode_AsUCS4(ustr, (Py_UCS4 *)self->ob_item + old_size, n, 0)) + return NULL; } Py_INCREF(Py_None); @@ -1545,14 +1554,14 @@ static PyObject * array_tounicode(arrayobject *self, PyObject *unused) { - char typecode; - typecode = self->ob_descr->typecode; - if ((typecode != 'u')) { + if (self->ob_descr->typecode != 'u') { PyErr_SetString(PyExc_ValueError, "tounicode() may only be called on unicode type arrays"); return NULL; } - return PyUnicode_FromUnicode((Py_UNICODE *) self->ob_item, Py_SIZE(self)); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (Py_UCS4 *) self->ob_item, + Py_SIZE(self)); } PyDoc_STRVAR(tounicode_doc, @@ -1659,13 +1668,7 @@ return UNSIGNED_INT8; case 'u': - if (sizeof(Py_UNICODE) == 2) { - return UTF16_LE + is_big_endian; - } - if (sizeof(Py_UNICODE) == 4) { - return UTF32_LE + is_big_endian; - } - return UNKNOWN_FORMAT; + return UTF32_LE + is_big_endian; case 'f': if (sizeof(float) == 4) { @@ -2411,14 +2414,8 @@ view->strides = &(view->itemsize); view->format = NULL; view->internal = NULL; - if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) view->format = self->ob_descr->formats; -#ifdef Py_UNICODE_WIDE - if (self->ob_descr->typecode == 'u') { - view->format = "w"; - } -#endif - } finish: self->ob_exports++; @@ -2543,7 +2540,7 @@ return NULL; } self->ob_item = item; - Py_SIZE(self) = n / sizeof(Py_UNICODE); + Py_SIZE(self) = n / sizeof(Py_UCS4); memcpy(item, PyUnicode_AS_DATA(initial), n); self->allocated = Py_SIZE(self); } -- Repository URL: http://hg.python.org/cpython

1 0

r88903 - tracker/instances/python-dev/schema.py
by ezio.melotti Sept. 29, 2011

Sept. 29, 2011

Author: ezio.melotti Date: Fri Sep 30 00:50:50 2011 New Revision: 88903 Log: #421: allow users to create issues with hgrepo links. Modified: tracker/instances/python-dev/schema.py Modified: tracker/instances/python-dev/schema.py ============================================================================== --- tracker/instances/python-dev/schema.py (original) +++ tracker/instances/python-dev/schema.py Fri Sep 30 00:50:50 2011 @@ -267,7 +267,7 @@ properties=('title', 'type', 'components', 'versions', 'severity', - 'messages', 'files', 'nosy'), + 'messages', 'files', 'nosy', 'hgrepos'), description='User can report and discuss issues') db.security.addPermissionToRole('User', p)

1 0

cpython: raw_unicode_escape() uses the new Unicode API
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/b9558df8cc58 changeset: 72539:b9558df8cc58 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 23:50:23 2011 +0200 summary: raw_unicode_escape() uses the new Unicode API files: Modules/_pickle.c | 75 +++++++++++----------------------- 1 files changed, 25 insertions(+), 50 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1769,33 +1769,37 @@ /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates backslash and newline characters to \uXXXX escapes. */ static PyObject * -raw_unicode_escape(const Py_UNICODE *s, Py_ssize_t size) -{ +raw_unicode_escape(PyObject *obj) +{ + static const char *hexdigits = "0123456789abcdef"; PyObject *repr, *result; char *p; - char *q; - - static const char *hexdigits = "0123456789abcdef"; - -#ifdef Py_UNICODE_WIDE - const Py_ssize_t expandsize = 10; -#else - const Py_ssize_t expandsize = 6; -#endif + Py_ssize_t i, size, expandsize; + void *data; + unsigned int kind; + + if (PyUnicode_READY(obj)) + return NULL; + + size = PyUnicode_GET_LENGTH(obj); + data = PyUnicode_DATA(obj); + kind = PyUnicode_KIND(obj); + if (kind == PyUnicode_4BYTE_KIND) + expandsize = 10; + else + expandsize = 6; if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - repr = PyByteArray_FromStringAndSize(NULL, expandsize * size); if (repr == NULL) return NULL; if (size == 0) goto done; - p = q = PyByteArray_AS_STRING(repr); - while (size-- > 0) { - Py_UNICODE ch = *s++; -#ifdef Py_UNICODE_WIDE + p = PyByteArray_AS_STRING(repr); + for (i=0; i < size; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { *p++ = '\\'; @@ -1809,36 +1813,8 @@ *p++ = hexdigits[(ch >> 4) & 0xf]; *p++ = hexdigits[ch & 15]; } - else -#else - /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ - if (ch >= 0xD800 && ch < 0xDC00) { - Py_UNICODE ch2; - Py_UCS4 ucs; - - ch2 = *s++; - size--; - if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0xf]; - *p++ = hexdigits[(ucs >> 24) & 0xf]; - *p++ = hexdigits[(ucs >> 20) & 0xf]; - *p++ = hexdigits[(ucs >> 16) & 0xf]; - *p++ = hexdigits[(ucs >> 12) & 0xf]; - *p++ = hexdigits[(ucs >> 8) & 0xf]; - *p++ = hexdigits[(ucs >> 4) & 0xf]; - *p++ = hexdigits[ucs & 0xf]; - continue; - } - /* Fall through: isolated surrogates are copied as-is */ - s--; - size++; - } -#endif /* Map 16-bit characters to '\uxxxx' */ - if (ch >= 256 || ch == '\\' || ch == '\n') { + else if (ch >= 256 || ch == '\\' || ch == '\n') { *p++ = '\\'; *p++ = 'u'; *p++ = hexdigits[(ch >> 12) & 0xf]; @@ -1850,9 +1826,9 @@ else *p++ = (char) ch; } - size = p - q; - - done: + size = p - PyByteArray_AS_STRING(repr); + +done: result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size); Py_DECREF(repr); return result; @@ -1893,8 +1869,7 @@ else { const char unicode_op = UNICODE; - encoded = raw_unicode_escape(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj)); + encoded = raw_unicode_escape(obj); if (encoded == NULL) goto error; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Strip trailing spaces of _pickle.c
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/a4a9ba2c31ef changeset: 72538:a4a9ba2c31ef user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 23:40:53 2011 +0200 summary: Strip trailing spaces of _pickle.c files: Modules/_pickle.c | 34 +++++++++++++++++----------------- 1 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -605,7 +605,7 @@ /*************************************************************************/ /* Helpers for creating the argument tuple passed to functions. This has the - performance advantage of calling PyTuple_New() only once. + performance advantage of calling PyTuple_New() only once. XXX(avassalotti): Inline directly in _Pickler_FastCall() and _Unpickler_FastCall(). */ @@ -813,7 +813,7 @@ fix_imports = PyObject_IsTrue(fix_imports_obj); if (fix_imports == -1) return -1; - + self->proto = proto; self->bin = proto > 0; self->fix_imports = fix_imports && proto < 3; @@ -909,7 +909,7 @@ Py_ssize_t read_size, prefetched_size = 0; assert(self->read != NULL); - + if (_Unpickler_SkipConsumed(self) < 0) return -1; @@ -1037,7 +1037,7 @@ self->next_read_idx = num_read; return _Unpickler_CopyLine(self, self->input_buffer, num_read, result); } - + /* If we get here, we've run off the end of the input string. Return the remaining string and let the caller figure it out. */ *result = self->input_buffer + self->next_read_idx; @@ -1601,7 +1601,7 @@ * bits. */ if (sign < 0 && - nbytes > 1 && + nbytes > 1 && pdata[nbytes - 1] == 0xff && (pdata[nbytes - 2] & 0x80) != 0) { nbytes--; @@ -1667,7 +1667,7 @@ return -1; if (_Pickler_Write(self, pdata, 9) < 0) return -1; - } + } else { int result = -1; char *buf = NULL; @@ -1782,10 +1782,10 @@ #else const Py_ssize_t expandsize = 6; #endif - + if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - + repr = PyByteArray_FromStringAndSize(NULL, expandsize * size); if (repr == NULL) return NULL; @@ -3055,7 +3055,7 @@ return -1; if (state) { - if (save(self, state, 0) < 0 || + if (save(self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -3309,7 +3309,7 @@ Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -3789,7 +3789,7 @@ 0, /*tp_is_gc*/ }; -/* Temporary helper for calling self.find_class(). +/* Temporary helper for calling self.find_class(). XXX: It would be nice to able to avoid Python function call overhead, by using directly the C version of find_class(), when find_class() is not @@ -3842,7 +3842,7 @@ return bad_readline(); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? + /* XXX: Should the base argument of strtol() be explicitly set to 10? XXX(avassalotti): Should this uses PyOS_strtol()? */ x = strtol(s, &endptr, 0); @@ -4207,7 +4207,7 @@ x = calc_binint(s, 4); if (x < 0) { - PyErr_SetString(UnpicklingError, + PyErr_SetString(UnpicklingError, "BINSTRING pickle has negative byte count"); return -1; } @@ -5001,7 +5001,7 @@ return stack_underflow(); if (len == x) /* nothing to do */ return 0; - if ((len - x) % 2 != 0) { + if ((len - x) % 2 != 0) { /* Currupt or hostile pickle -- we never write one like this. */ PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS"); return -1; @@ -5358,7 +5358,7 @@ not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (self->read == NULL) { - PyErr_Format(UnpicklingError, + PyErr_Format(UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -5460,7 +5460,7 @@ global = PyObject_GetAttr(module, global_name); Py_DECREF(module); } - else { + else { global = PyObject_GetAttr(module, global_name); } return global; @@ -5637,7 +5637,7 @@ * intentional, as these should be treated as black-box implementation details. * * We do, however, have to implement pickling/unpickling support because of - * real-world code like cvs2svn. + * real-world code like cvs2svn. */ typedef struct { -- Repository URL: http://hg.python.org/cpython

1 0

cpython: PyLocale_strxfrm() uses the new Unicode API
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/771064b4466f changeset: 72537:771064b4466f user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 23:32:06 2011 +0200 summary: PyLocale_strxfrm() uses the new Unicode API files: Modules/_localemodule.c | 39 +++++++++++----------------- 1 files changed, 15 insertions(+), 24 deletions(-) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -271,31 +271,22 @@ static PyObject* PyLocale_strxfrm(PyObject* self, PyObject* args) { - Py_UNICODE *s0; - Py_ssize_t n0; - wchar_t *s, *buf = NULL; - size_t n1, n2; + PyObject *str; + Py_ssize_t n1; + wchar_t *s = NULL, *buf = NULL; + size_t n2; PyObject *result = NULL; -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T - Py_ssize_t i; -#endif - if (!PyArg_ParseTuple(args, "u#:strxfrm", &s0, &n0)) + if (!PyArg_ParseTuple(args, "U:strxfrm", &str)) return NULL; -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - s = (wchar_t *) s0; -#else - s = PyMem_Malloc((n0+1)*sizeof(wchar_t)); - if (!s) - return PyErr_NoMemory(); - for (i=0; i<=n0; i++) - s[i] = s0[i]; -#endif + s = PyUnicode_AsWideCharString(str, &n1); + if (s == NULL) + goto exit; /* assume no change in size, first */ - n1 = wcslen(s) + 1; - buf = PyMem_Malloc(n1*sizeof(wchar_t)); + n1 = n1 + 1; + buf = PyMem_Malloc(n1 * sizeof(wchar_t)); if (!buf) { PyErr_NoMemory(); goto exit; @@ -311,11 +302,11 @@ n2 = wcsxfrm(buf, s, n2+1); } result = PyUnicode_FromWideChar(buf, n2); - exit: - if (buf) PyMem_Free(buf); -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T - PyMem_Free(s); -#endif +exit: + if (buf) + PyMem_Free(buf); + if (s) + PyMem_Free(s); return result; } #endif -- Repository URL: http://hg.python.org/cpython

1 0

cpython: fileio_init() checks for failure on conversion to Py_UNICODE*
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/06d9652dbd67 changeset: 72536:06d9652dbd67 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 23:19:04 2011 +0200 summary: fileio_init() checks for failure on conversion to Py_UNICODE* files: Modules/_io/fileio.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -259,9 +259,11 @@ } #ifdef MS_WINDOWS - if (PyUnicode_Check(nameobj)) - widename = PyUnicode_AS_UNICODE(nameobj); - if (widename == NULL) + if (PyUnicode_Check(nameobj)) { + widename = PyUnicode_AsUnicode(nameobj); + if (widename == NULL) + return -1; + } else #endif if (fd < 0) { @@ -378,7 +380,7 @@ if (self->fd < 0) { #ifdef MS_WINDOWS if (widename != NULL) - PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename); + PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); else #endif PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Fix test_codeccallbacks for Windows: check size of wchar_t, not sys.maxunicode
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/7a50e549bd11 changeset: 72535:7a50e549bd11 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 20:01:55 2011 +0200 summary: Fix test_codeccallbacks for Windows: check size of wchar_t, not sys.maxunicode files: Lib/test/test_codeccallbacks.py | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -2,6 +2,8 @@ import sys, codecs, html.entities, unicodedata import ctypes +SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) + class PosReturn: # this can be used for configurable callbacks @@ -206,7 +208,7 @@ b"\x00\x00\x00\x00\x00".decode, "unicode-internal", ) - if sys.maxunicode > 0xffff: + if SIZEOF_WCHAR_T == 4: def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) @@ -356,7 +358,7 @@ ["ascii", "\uffffx", 0, 1, "ouch"], "'ascii' codec can't encode character '\\uffff' in position 0: ouch" ) - if sys.maxunicode > 0xffff: + if SIZEOF_WCHAR_T == 4: self.check_exceptionobjectargs( UnicodeEncodeError, ["ascii", "\U00010000x", 0, 1, "ouch"], @@ -391,7 +393,7 @@ ["g\uffffrk", 1, 2, "ouch"], "can't translate character '\\uffff' in position 1: ouch" ) - if sys.maxunicode > 0xffff: + if SIZEOF_WCHAR_T == 4: self.check_exceptionobjectargs( UnicodeTranslateError, ["g\U00010000rk", 1, 2, "ouch"], @@ -682,7 +684,7 @@ # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors() # and inline implementations v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000) - if sys.maxunicode>=100000: + if SIZEOF_WCHAR_T == 4: v += (100000, 500000, 1000000) s = "".join([chr(x) for x in v]) codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Fix test_codecs for Windows: check size of wchar_t, not sys.maxunicode
by victor.stinner Sept. 29, 2011

Sept. 29, 2011

http://hg.python.org/cpython/rev/85d11cf67aa8 changeset: 72534:85d11cf67aa8 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 19:53:55 2011 +0200 summary: Fix test_codecs for Windows: check size of wchar_t, not sys.maxunicode files: Lib/test/test_codecs.py | 83 +++++++++++++++------------- 1 files changed, 43 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3,6 +3,9 @@ import codecs import locale import sys, _testcapi, io +import ctypes + +SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) class Queue(object): """ @@ -888,53 +891,53 @@ self.assertEqual(uni, puny.decode("punycode")) class UnicodeInternalTest(unittest.TestCase): + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): # Decoding with unicode_internal used to not correctly handle "code # points" above 0x10ffff on UCS-4 builds. - if sys.maxunicode > 0xffff: - ok = [ - (b"\x00\x10\xff\xff", "\U0010ffff"), - (b"\x00\x00\x01\x01", "\U00000101"), - (b"", ""), - ] - not_ok = [ - b"\x7f\xff\xff\xff", - b"\x80\x00\x00\x00", - b"\x81\x00\x00\x00", - b"\x00", - b"\x00\x00\x00\x00\x00", - ] - for internal, uni in ok: - if sys.byteorder == "little": - internal = bytes(reversed(internal)) - self.assertEqual(uni, internal.decode("unicode_internal")) - for internal in not_ok: - if sys.byteorder == "little": - internal = bytes(reversed(internal)) - self.assertRaises(UnicodeDecodeError, internal.decode, - "unicode_internal") + ok = [ + (b"\x00\x10\xff\xff", "\U0010ffff"), + (b"\x00\x00\x01\x01", "\U00000101"), + (b"", ""), + ] + not_ok = [ + b"\x7f\xff\xff\xff", + b"\x80\x00\x00\x00", + b"\x81\x00\x00\x00", + b"\x00", + b"\x00\x00\x00\x00\x00", + ] + for internal, uni in ok: + if sys.byteorder == "little": + internal = bytes(reversed(internal)) + self.assertEqual(uni, internal.decode("unicode_internal")) + for internal in not_ok: + if sys.byteorder == "little": + internal = bytes(reversed(internal)) + self.assertRaises(UnicodeDecodeError, internal.decode, + "unicode_internal") + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_decode_error_attributes(self): - if sys.maxunicode > 0xffff: - try: - b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") - except UnicodeDecodeError as ex: - self.assertEqual("unicode_internal", ex.encoding) - self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) - self.assertEqual(4, ex.start) - self.assertEqual(8, ex.end) - else: - self.fail() + try: + b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") + except UnicodeDecodeError as ex: + self.assertEqual("unicode_internal", ex.encoding) + self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) + self.assertEqual(4, ex.start) + self.assertEqual(8, ex.end) + else: + self.fail() + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_decode_callback(self): - if sys.maxunicode > 0xffff: - codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) - decoder = codecs.getdecoder("unicode_internal") - ab = "ab".encode("unicode_internal").decode() - ignored = decoder(bytes("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), - "ascii"), - "UnicodeInternalTest") - self.assertEqual(("ab", 12), ignored) + codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) + decoder = codecs.getdecoder("unicode_internal") + ab = "ab".encode("unicode_internal").decode() + ignored = decoder(bytes("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), + "ascii"), + "UnicodeInternalTest") + self.assertEqual(("ab", 12), ignored) def test_encode_length(self): # Issue 3739 -- Repository URL: http://hg.python.org/cpython

1 0