Mailman 3 September 2011 - Python-checkins

cpython (merge 3.2 -> default): Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas
by charles-francois.natali 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/d8dc24c4e83d changeset: 72532:d8dc24c4e83d parent: 72527:a11970b158f2 parent: 72531:160b52c9e8b3 user: Charles-François Natali <neologix(a)free.fr> date: Thu Sep 29 19:49:37 2011 +0200 summary: Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas Jarosch. files: Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/ossaudiodev.c | 1 + 3 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -463,6 +463,7 @@ Geert Jansen Jack Jansen Bill Janssen +Thomas Jarosch Julien Jehannet Drew Jenkins Flemming Kjær Jensen diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1298,6 +1298,9 @@ Extension Modules ----------------- +- Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by + Thomas Jarosch. + - Issue #13013: ctypes: Fix a reference leak in PyCArrayType_from_ctype. Thanks to Suman Saha for finding the bug and providing a patch. diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -129,6 +129,7 @@ } if (ioctl(fd, SNDCTL_DSP_GETFMTS, &afmts) == -1) { + close(fd); PyErr_SetFromErrnoWithFilename(PyExc_IOError, devicename); return NULL; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge default -> default): Merge.
by charles-francois.natali 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/f7de67283c94 changeset: 72533:f7de67283c94 parent: 72532:d8dc24c4e83d parent: 72529:2fc483009f6d user: Charles-François Natali <neologix(a)free.fr> date: Thu Sep 29 19:51:46 2011 +0200 summary: Merge. files: Include/unicodeobject.h | 26 +++++---------- Objects/unicodeobject.c | 48 ++++++++++++++-------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -377,16 +377,6 @@ PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \ _PyUnicode_NONCOMPACT_DATA(op)) -#define _PyUnicode_UTF8(op) \ - (PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((char*)((PyASCIIObject*)(op) + 1)) : \ - ((PyCompactUnicodeObject*)(op))->utf8) - -#define _PyUnicode_UTF8_LENGTH(op) \ - (PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((PyASCIIObject*)(op))->length : \ - ((PyCompactUnicodeObject*)(op))->utf8_length) - /* Compute (index * char_size) where char_size is 2 ** (kind - 1). The index is a character index, the result is a size in bytes. */ @@ -466,7 +456,7 @@ #define PyUnicode_READY(op) \ (assert(PyUnicode_Check(op)), \ (PyUnicode_IS_READY(op) ? \ - 0 : _PyUnicode_Ready((PyUnicodeObject *)(op)))) + 0 : _PyUnicode_Ready((PyObject *)(op)))) /* Return a maximum character value which is suitable for creating another string based on op. This is always an approximation but more efficient @@ -507,14 +497,16 @@ ); #endif -/* Initializes the canonical string representation from a the deprected - wstr/Py_UNICODE representation. This function is used to convert - unicode objects which were created using the old API to the new flexible - format introduced with PEP 393. The PyUnicode_READY() macro can be - more efficient if the string is already ready. */ +/* Initializes the canonical string representation from a the deprecated + wstr/Py_UNICODE representation. This function is used to convert Unicode + objects which were created using the old API to the new flexible format + introduced with PEP 393. + + Don't call this function directly, use the public PyUnicode_READY() macro + instead. */ #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyUnicode_Ready( - PyUnicodeObject *unicode /* Unicode object */ + PyObject *unicode /* Unicode object */ ); #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -105,6 +105,14 @@ } \ } while (0) +#define _PyUnicode_UTF8(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((char*)((PyASCIIObject*)(op) + 1)) : \ + ((PyCompactUnicodeObject*)(op))->utf8) +#define _PyUnicode_UTF8_LENGTH(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((PyASCIIObject*)(op))->length : \ + ((PyCompactUnicodeObject*)(op))->utf8_length) #define _PyUnicode_WSTR(op) (((PyASCIIObject*)(op))->wstr) #define _PyUnicode_WSTR_LENGTH(op) (((PyCompactUnicodeObject*)(op))->wstr_length) #define _PyUnicode_LENGTH(op) (((PyASCIIObject *)(op))->length) @@ -773,8 +781,9 @@ #endif int -_PyUnicode_Ready(PyUnicodeObject *unicode) -{ +_PyUnicode_Ready(PyObject *obj) +{ + PyUnicodeObject *unicode = (PyUnicodeObject *)obj; wchar_t *end; Py_UCS4 maxchar = 0; Py_ssize_t num_surrogates; @@ -782,25 +791,19 @@ Py_ssize_t length_wo_surrogates; #endif - assert(PyUnicode_Check(unicode)); - - if (unicode->data.any != NULL) { - assert(PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND); - return 0; - } - /* _PyUnicode_Ready() is only intented for old-style API usage where - * strings were created using _PyObject_New() and where no canonical - * representation (the str field) has been set yet aka strings - * which are not yet ready. - */ + strings were created using _PyObject_New() and where no canonical + representation (the str field) has been set yet aka strings + which are not yet ready. */ + assert(PyUnicode_Check(obj)); + assert(!PyUnicode_IS_READY(obj)); + assert(!PyUnicode_IS_COMPACT(obj)); + assert(_PyUnicode_KIND(obj) == PyUnicode_WCHAR_KIND); assert(_PyUnicode_WSTR(unicode) != NULL); - assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND); - assert(!PyUnicode_IS_COMPACT(unicode)); - assert(!PyUnicode_IS_READY(unicode)); + assert(unicode->data.any == NULL); + assert(unicode->_base.utf8 == NULL); /* Actually, it should neither be interned nor be anything else: */ - assert(_PyUnicode_STATE(unicode).interned == 0); - assert(unicode->_base.utf8 == NULL); + assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED); #ifdef Py_DEBUG ++unicode_ready_calls; @@ -808,11 +811,8 @@ end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode); if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end, - &maxchar, - &num_surrogates) == -1) { - assert(0 && "PyUnicode_FindMaxCharAndNumSurrogatePairs failed"); + &maxchar, &num_surrogates) == -1) return -1; - } if (maxchar < 256) { unicode->data.any = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1); @@ -1038,8 +1038,8 @@ /* If not empty and not single character, copy the Unicode data into the new object */ - if (find_maxchar_surrogates(u, u + size, &maxchar, - &num_surrogates) == -1) + if (find_maxchar_surrogates(u, u + size, + &maxchar, &num_surrogates) == -1) return NULL; unicode = (PyUnicodeObject *) PyUnicode_New(size - num_surrogates, -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.2): Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas
by charles-francois.natali 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/160b52c9e8b3 changeset: 72531:160b52c9e8b3 branch: 3.2 parent: 72473:58a75eeb5c8e user: Charles-François Natali <neologix(a)free.fr> date: Thu Sep 29 19:46:37 2011 +0200 summary: Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas Jarosch. files: Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/ossaudiodev.c | 1 + 3 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -436,6 +436,7 @@ Geert Jansen Jack Jansen Bill Janssen +Thomas Jarosch Drew Jenkins Flemming Kjær Jensen MunSic Jeong diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -82,6 +82,9 @@ Extension Modules ----------------- +- Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by + Thomas Jarosch. + - Issue #13013: ctypes: Fix a reference leak in PyCArrayType_from_ctype. Thanks to Suman Saha for finding the bug and providing a patch. diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -129,6 +129,7 @@ } if (ioctl(fd, SNDCTL_DSP_GETFMTS, &afmts) == -1) { + close(fd); PyErr_SetFromErrnoWithFilename(PyExc_IOError, devicename); return NULL; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas
by charles-francois.natali 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/dec00ae64ca8 changeset: 72530:dec00ae64ca8 branch: 2.7 parent: 72513:f612b783bc17 user: Charles-François Natali <neologix(a)free.fr> date: Thu Sep 29 19:43:01 2011 +0200 summary: Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by Thomas Jarosch. files: Misc/ACKS | 1 + Misc/NEWS | 3 +++ Modules/ossaudiodev.c | 1 + 3 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -402,6 +402,7 @@ Geert Jansen Jack Jansen Bill Janssen +Thomas Jarosch Drew Jenkins Flemming Kjær Jensen Jiba diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -50,6 +50,9 @@ Library ------- +- Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by + Thomas Jarosch. + - Issue #12931: xmlrpclib now encodes Unicode URI to ISO-8859-1, instead of failing with a UnicodeDecodeError. diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -129,6 +129,7 @@ } if (ioctl(fd, SNDCTL_DSP_GETFMTS, &afmts) == -1) { + close(fd); PyErr_SetFromErrnoWithFilename(PyExc_IOError, devicename); return NULL; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython: _PyUnicode_Ready() cannot be used on ready strings anymore
by victor.stinner 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/2fc483009f6d changeset: 72529:2fc483009f6d user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 19:43:17 2011 +0200 summary: _PyUnicode_Ready() cannot be used on ready strings anymore * Change its prototype: PyObject* instead of PyUnicodeoObject*. * Remove an old assertion, the result of PyUnicode_READY (_PyUnicode_Ready) must be checked instead files: Include/unicodeobject.h | 16 ++++++----- Objects/unicodeobject.c | 40 +++++++++++----------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -456,7 +456,7 @@ #define PyUnicode_READY(op) \ (assert(PyUnicode_Check(op)), \ (PyUnicode_IS_READY(op) ? \ - 0 : _PyUnicode_Ready((PyUnicodeObject *)(op)))) + 0 : _PyUnicode_Ready((PyObject *)(op)))) /* Return a maximum character value which is suitable for creating another string based on op. This is always an approximation but more efficient @@ -497,14 +497,16 @@ ); #endif -/* Initializes the canonical string representation from a the deprected - wstr/Py_UNICODE representation. This function is used to convert - unicode objects which were created using the old API to the new flexible - format introduced with PEP 393. The PyUnicode_READY() macro can be - more efficient if the string is already ready. */ +/* Initializes the canonical string representation from a the deprecated + wstr/Py_UNICODE representation. This function is used to convert Unicode + objects which were created using the old API to the new flexible format + introduced with PEP 393. + + Don't call this function directly, use the public PyUnicode_READY() macro + instead. */ #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyUnicode_Ready( - PyUnicodeObject *unicode /* Unicode object */ + PyObject *unicode /* Unicode object */ ); #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -781,8 +781,9 @@ #endif int -_PyUnicode_Ready(PyUnicodeObject *unicode) -{ +_PyUnicode_Ready(PyObject *obj) +{ + PyUnicodeObject *unicode = (PyUnicodeObject *)obj; wchar_t *end; Py_UCS4 maxchar = 0; Py_ssize_t num_surrogates; @@ -790,25 +791,19 @@ Py_ssize_t length_wo_surrogates; #endif - assert(PyUnicode_Check(unicode)); - - if (unicode->data.any != NULL) { - assert(PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND); - return 0; - } - /* _PyUnicode_Ready() is only intented for old-style API usage where - * strings were created using _PyObject_New() and where no canonical - * representation (the str field) has been set yet aka strings - * which are not yet ready. - */ + strings were created using _PyObject_New() and where no canonical + representation (the str field) has been set yet aka strings + which are not yet ready. */ + assert(PyUnicode_Check(obj)); + assert(!PyUnicode_IS_READY(obj)); + assert(!PyUnicode_IS_COMPACT(obj)); + assert(_PyUnicode_KIND(obj) == PyUnicode_WCHAR_KIND); assert(_PyUnicode_WSTR(unicode) != NULL); - assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND); - assert(!PyUnicode_IS_COMPACT(unicode)); - assert(!PyUnicode_IS_READY(unicode)); + assert(unicode->data.any == NULL); + assert(unicode->_base.utf8 == NULL); /* Actually, it should neither be interned nor be anything else: */ - assert(_PyUnicode_STATE(unicode).interned == 0); - assert(unicode->_base.utf8 == NULL); + assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED); #ifdef Py_DEBUG ++unicode_ready_calls; @@ -816,11 +811,8 @@ end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode); if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end, - &maxchar, - &num_surrogates) == -1) { - assert(0 && "PyUnicode_FindMaxCharAndNumSurrogatePairs failed"); + &maxchar, &num_surrogates) == -1) return -1; - } if (maxchar < 256) { unicode->data.any = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1); @@ -1046,8 +1038,8 @@ /* If not empty and not single character, copy the Unicode data into the new object */ - if (find_maxchar_surrogates(u, u + size, &maxchar, - &num_surrogates) == -1) + if (find_maxchar_surrogates(u, u + size, + &maxchar, &num_surrogates) == -1) return NULL; unicode = (PyUnicodeObject *) PyUnicode_New(size - num_surrogates, -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Move _PyUnicode_UTF8() and _PyUnicode_UTF8_LENGTH() outside unicodeobject.h
by victor.stinner 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/8cebe7310db7 changeset: 72528:8cebe7310db7 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Thu Sep 29 19:31:34 2011 +0200 summary: Move _PyUnicode_UTF8() and _PyUnicode_UTF8_LENGTH() outside unicodeobject.h Move these macros to unicodeobject.c files: Include/unicodeobject.h | 10 ---------- Objects/unicodeobject.c | 8 ++++++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -377,16 +377,6 @@ PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \ _PyUnicode_NONCOMPACT_DATA(op)) -#define _PyUnicode_UTF8(op) \ - (PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((char*)((PyASCIIObject*)(op) + 1)) : \ - ((PyCompactUnicodeObject*)(op))->utf8) - -#define _PyUnicode_UTF8_LENGTH(op) \ - (PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((PyASCIIObject*)(op))->length : \ - ((PyCompactUnicodeObject*)(op))->utf8_length) - /* Compute (index * char_size) where char_size is 2 ** (kind - 1). The index is a character index, the result is a size in bytes. */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -105,6 +105,14 @@ } \ } while (0) +#define _PyUnicode_UTF8(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((char*)((PyASCIIObject*)(op) + 1)) : \ + ((PyCompactUnicodeObject*)(op))->utf8) +#define _PyUnicode_UTF8_LENGTH(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((PyASCIIObject*)(op))->length : \ + ((PyCompactUnicodeObject*)(op))->utf8_length) #define _PyUnicode_WSTR(op) (((PyASCIIObject*)(op))->wstr) #define _PyUnicode_WSTR_LENGTH(op) (((PyCompactUnicodeObject*)(op))->wstr_length) #define _PyUnicode_LENGTH(op) (((PyASCIIObject *)(op))->length) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Add a note in PyUnicode_CopyCharacters() doc: it doesn't write null character
by victor.stinner 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/a11970b158f2 changeset: 72527:a11970b158f2 user: Victor Stinner <vstinner(a)wyplay.com> date: Thu Sep 29 14:14:38 2011 +0200 summary: Add a note in PyUnicode_CopyCharacters() doc: it doesn't write null character Cleanup also the code (avoid the goto). files: Include/unicodeobject.h | 8 +- Objects/unicodeobject.c | 87 ++++++++++++++-------------- 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -521,9 +521,9 @@ /* Copy character from one unicode object into another, this function performs character conversion when necessary and falls back to memcpy if possible. - Fail if 'to' is smaller than how_many or smaller than len(from)-from_start, - or if kind(from[from_start:from_start+how_many]) > kind(to), or if to has - more than 1 reference. + Fail if to is too small (smaller than how_many or smaller than + len(from)-from_start), or if kind(from[from_start:from_start+how_many]) > + kind(to), or if to has more than 1 reference. Return the number of written character, or return -1 and raise an exception on error. @@ -533,6 +533,8 @@ how_many = min(how_many, len(from) - from_start) to[to_start:to_start+how_many] = from[from_start:from_start+how_many] return how_many + + Note: The function doesn't write a terminating null character. */ #ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -615,8 +615,8 @@ PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) { - unsigned int from_kind; - unsigned int to_kind; + unsigned int from_kind, to_kind; + void *from_data, *to_data; assert(PyUnicode_Check(from)); assert(PyUnicode_Check(to)); @@ -645,44 +645,20 @@ _PyUnicode_DIRTY(to); from_kind = PyUnicode_KIND(from); + from_data = PyUnicode_DATA(from); to_kind = PyUnicode_KIND(to); + to_data = PyUnicode_DATA(to); if (from_kind == to_kind) { /* fast path */ - Py_MEMCPY((char*)PyUnicode_DATA(to) + Py_MEMCPY((char*)to_data + PyUnicode_KIND_SIZE(to_kind, to_start), - (char*)PyUnicode_DATA(from) + (char*)from_data + PyUnicode_KIND_SIZE(from_kind, from_start), PyUnicode_KIND_SIZE(to_kind, how_many)); - return how_many; - } - - if (from_kind > to_kind) { - /* slow path to check for character overflow */ - const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); - void *from_data = PyUnicode_DATA(from); - void *to_data = PyUnicode_DATA(to); - Py_UCS4 ch, maxchar; - Py_ssize_t i; - int overflow; - - maxchar = 0; - overflow = 0; - for (i=0; i < how_many; i++) { - ch = PyUnicode_READ(from_kind, from_data, from_start + i); - if (ch > maxchar) { - maxchar = ch; - if (maxchar > to_maxchar) { - overflow = 1; - break; - } - } - PyUnicode_WRITE(to_kind, to_data, to_start + i, ch); - } - if (!overflow) - return how_many; - } - else if (from_kind == PyUnicode_1BYTE_KIND && to_kind == PyUnicode_2BYTE_KIND) + } + else if (from_kind == PyUnicode_1BYTE_KIND + && to_kind == PyUnicode_2BYTE_KIND) { _PyUnicode_CONVERT_BYTES( Py_UCS1, Py_UCS2, @@ -690,7 +666,6 @@ PyUnicode_1BYTE_DATA(from) + from_start + how_many, PyUnicode_2BYTE_DATA(to) + to_start ); - return how_many; } else if (from_kind == PyUnicode_1BYTE_KIND && to_kind == PyUnicode_4BYTE_KIND) @@ -701,7 +676,6 @@ PyUnicode_1BYTE_DATA(from) + from_start + how_many, PyUnicode_4BYTE_DATA(to) + to_start ); - return how_many; } else if (from_kind == PyUnicode_2BYTE_KIND && to_kind == PyUnicode_4BYTE_KIND) @@ -712,14 +686,41 @@ PyUnicode_2BYTE_DATA(from) + from_start + how_many, PyUnicode_4BYTE_DATA(to) + to_start ); - return how_many; - } - PyErr_Format(PyExc_ValueError, - "Cannot copy UCS%u characters " - "into a string of UCS%u characters", - 1 << (from_kind - 1), - 1 << (to_kind -1)); - return -1; + } + else { + int invalid_kinds; + if (from_kind > to_kind) { + /* slow path to check for character overflow */ + const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); + Py_UCS4 ch, maxchar; + Py_ssize_t i; + + maxchar = 0; + invalid_kinds = 0; + for (i=0; i < how_many; i++) { + ch = PyUnicode_READ(from_kind, from_data, from_start + i); + if (ch > maxchar) { + maxchar = ch; + if (maxchar > to_maxchar) { + invalid_kinds = 1; + break; + } + } + PyUnicode_WRITE(to_kind, to_data, to_start + i, ch); + } + } + else + invalid_kinds = 1; + if (invalid_kinds) { + PyErr_Format(PyExc_ValueError, + "Cannot copy UCS%u characters " + "into a string of UCS%u characters", + 1 << (from_kind - 1), + 1 << (to_kind -1)); + return -1; + } + } + return how_many; } /* Find the maximum code point and count the number of surrogate pairs so a -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Re-enable test.
by martin.v.loewis 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/50f008019213 changeset: 72526:50f008019213 user: Martin v. Löwis <martin(a)v.loewis.de> date: Thu Sep 29 13:49:10 2011 +0200 summary: Re-enable test. files: Lib/test/test_pep3131.py | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pep3131.py b/Lib/test/test_pep3131.py --- a/Lib/test/test_pep3131.py +++ b/Lib/test/test_pep3131.py @@ -17,10 +17,7 @@ def test_non_bmp_normalized(self): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 - try: - self.assertIn("Unicode", dir()) - except AssertionError: - raise unittest._ExpectedFailure("doesn't work yet") + self.assertIn("Unicode", dir()) def test_invalid(self): try: -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Port normalization to new API.
by martin.v.loewis 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/40cf7204ca2b changeset: 72525:40cf7204ca2b user: Martin v. Löwis <martin(a)v.loewis.de> date: Thu Sep 29 13:39:38 2011 +0200 summary: Port normalization to new API. files: Modules/unicodedata.c | 188 ++++++++++++++++++----------- 1 files changed, 118 insertions(+), 70 deletions(-) diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -494,36 +494,44 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; - Py_UNICODE *i, *end, *o; + Py_UCS4 *output; + Py_ssize_t i, o, osize; + int kind; + void *data; /* Longest decomposition in Unicode 3.2: U+FDFA */ - Py_UNICODE stack[20]; + Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; stackptr = 0; - isize = PyUnicode_GET_SIZE(input); + isize = PyUnicode_GET_LENGTH(input); /* Overallocate atmost 10 characters. */ space = (isize > 10 ? 10 : isize) + isize; - result = PyUnicode_FromUnicode(NULL, space); - if (!result) + osize = space; + output = PyMem_Malloc(space * sizeof(Py_UCS4)); + if (!output) { + PyErr_NoMemory(); return NULL; - i = PyUnicode_AS_UNICODE(input); - end = i + isize; - o = PyUnicode_AS_UNICODE(result); + } + i = o = 0; + kind = PyUnicode_KIND(input); + data = PyUnicode_DATA(input); - while (i < end) { - stack[stackptr++] = *i++; + while (i < isize) { + stack[stackptr++] = PyUnicode_READ(kind, data, i++); while(stackptr) { - Py_UNICODE code = stack[--stackptr]; + Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in a single step, so we need atleast that much room. */ if (space < 3) { - Py_ssize_t newsize = PyUnicode_GET_SIZE(result) + 10; + osize += 10; space += 10; - if (PyUnicode_Resize(&result, newsize) == -1) + output = PyMem_Realloc(output, osize*sizeof(Py_UCS4)); + if (output == NULL) { + PyErr_NoMemory(); return NULL; - o = PyUnicode_AS_UNICODE(result) + newsize - space; + } } /* Hangul Decomposition. */ if (SBase <= code && code < (SBase+SCount)) { @@ -531,11 +539,11 @@ int L = LBase + SIndex / NCount; int V = VBase + (SIndex % NCount) / TCount; int T = TBase + SIndex % TCount; - *o++ = L; - *o++ = V; + output[o++] = L; + output[o++] = V; space -= 2; if (T != TBase) { - *o++ = T; + output[o++] = T; space --; } continue; @@ -555,7 +563,7 @@ /* Copy character if it is not decomposable, or has a compatibility decomposition, but we do NFD. */ if (!count || (prefix && !k)) { - *o++ = code; + output[o++] = code; space--; continue; } @@ -568,15 +576,20 @@ } } - /* Drop overallocation. Cannot fail. */ - PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + output, o); + PyMem_Free(output); + if (!result) + return NULL; + /* result is guaranteed to be ready, as it is compact. */ + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); /* Sort canonically. */ - i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; - end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; + i = 0; + prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + for (i++; i < PyUnicode_GET_LENGTH(result); i++) { + cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; if (prev == 0 || cur == 0 || prev <= cur) { prev = cur; continue; @@ -584,23 +597,24 @@ /* Non-canonical order. Need to switch *i with previous. */ o = i - 1; while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; + Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); + PyUnicode_WRITE(kind, data, o+1, + PyUnicode_READ(kind, data, o)); + PyUnicode_WRITE(kind, data, o, tmp); o--; - if (o < PyUnicode_AS_UNICODE(result)) + if (o < 0) break; - prev = _getrecord_ex(*o)->combining; + prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; if (prev == 0 || prev <= cur) break; } - prev = _getrecord_ex(*i)->combining; + prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; } return result; } static int -find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code) +find_nfc_index(PyObject *self, struct reindex* nfc, Py_UCS4 code) { int index; for (index = 0; nfc[index].start; index++) { @@ -619,27 +633,36 @@ nfc_nfkc(PyObject *self, PyObject *input, int k) { PyObject *result; - Py_UNICODE *i, *i1, *o, *end; + int kind; + void *data; + Py_UCS4 *output; + Py_ssize_t i, i1, o, len; int f,l,index,index1,comb; - Py_UNICODE code; - Py_UNICODE *skipped[20]; + Py_UCS4 code; + Py_ssize_t skipped[20]; int cskipped = 0; result = nfd_nfkd(self, input, k); if (!result) return NULL; + /* result will be "ready". */ + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + len = PyUnicode_GET_LENGTH(result); - /* We are going to modify result in-place. - If nfd_nfkd is changed to sometimes return the input, - this code needs to be reviewed. */ - assert(result != input); - - i = PyUnicode_AS_UNICODE(result); - end = i + PyUnicode_GET_SIZE(result); - o = PyUnicode_AS_UNICODE(result); + /* We allocate a buffer for the output. + If we find that we made no changes, we still return + the NFD result. */ + output = PyMem_Malloc(len * sizeof(Py_UCS4)); + if (!output) { + PyErr_NoMemory(); + Py_DECREF(result); + return 0; + } + i = o = 0; again: - while (i < end) { + while (i < len) { for (index = 0; index < cskipped; index++) { if (skipped[index] == i) { /* *i character is skipped. @@ -652,33 +675,41 @@ } /* Hangul Composition. We don't need to check for <LV,T> pairs, since we always have decomposed data. */ - if (LBase <= *i && *i < (LBase+LCount) && - i + 1 < end && - VBase <= i[1] && i[1] <= (VBase+VCount)) { + code = PyUnicode_READ(kind, data, i); + if (LBase <= code && code < (LBase+LCount) && + i + 1 < len && + VBase <= PyUnicode_READ(kind, data, i+1) && + PyUnicode_READ(kind, data, i+1) <= (VBase+VCount)) { int LIndex, VIndex; - LIndex = i[0] - LBase; - VIndex = i[1] - VBase; + LIndex = code - LBase; + VIndex = PyUnicode_READ(kind, data, i+1) - VBase; code = SBase + (LIndex*VCount+VIndex)*TCount; i+=2; - if (i < end && - TBase <= *i && *i <= (TBase+TCount)) { - code += *i-TBase; + if (i < len && + TBase <= PyUnicode_READ(kind, data, i) && + PyUnicode_READ(kind, data, i) <= (TBase+TCount)) { + code += PyUnicode_READ(kind, data, i)-TBase; i++; } - *o++ = code; + output[o++] = code; continue; } - f = find_nfc_index(self, nfc_first, *i); + /* code is still input[i] here */ + f = find_nfc_index(self, nfc_first, code); if (f == -1) { - *o++ = *i++; + output[o++] = code; + i++; continue; } /* Find next unblocked character. */ i1 = i+1; comb = 0; - while (i1 < end) { - int comb1 = _getrecord_ex(*i1)->combining; + /* output base character for now; might be updated later. */ + output[o] = PyUnicode_READ(kind, data, i); + while (i1 < len) { + Py_UCS4 code1 = PyUnicode_READ(kind, data, i1); + int comb1 = _getrecord_ex(code1)->combining; if (comb) { if (comb1 == 0) break; @@ -688,8 +719,8 @@ continue; } } - l = find_nfc_index(self, nfc_last, *i1); - /* *i1 cannot be combined with *i. If *i1 + l = find_nfc_index(self, nfc_last, code1); + /* i1 cannot be combined with i. If i1 is a starter, we don't need to look further. Otherwise, record the combining class. */ if (l == -1) { @@ -708,19 +739,28 @@ goto not_combinable; /* Replace the original character. */ - *i = code; + output[o] = code; /* Mark the second character unused. */ assert(cskipped < 20); skipped[cskipped++] = i1; i1++; - f = find_nfc_index(self, nfc_first, *i); + f = find_nfc_index(self, nfc_first, output[o]); if (f == -1) break; } - *o++ = *i++; + /* Output character was already written. + Just advance the indices. */ + o++; i++; } - if (o != end) - PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result)); + if (o == len) { + /* No changes. Return original string. */ + PyMem_Free(output); + return result; + } + Py_DECREF(result); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + output, o); + PyMem_Free(output); return result; } @@ -728,7 +768,9 @@ static int is_normalized(PyObject *self, PyObject *input, int nfc, int k) { - Py_UNICODE *i, *end; + Py_ssize_t i, len; + int kind; + void *data; unsigned char prev_combining = 0, quickcheck_mask; /* An older version of the database is requested, quickchecks must be @@ -740,10 +782,13 @@ as described in http://unicode.org/reports/tr15/#Annex8. */ quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0)); - i = PyUnicode_AS_UNICODE(input); - end = i + PyUnicode_GET_SIZE(input); - while (i < end) { - const _PyUnicode_DatabaseRecord *record = _getrecord_ex(*i++); + i = 0; + kind = PyUnicode_KIND(input); + data = PyUnicode_DATA(input); + len = PyUnicode_GET_LENGTH(input); + while (i < len) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i++); + const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch); unsigned char combining = record->combining; unsigned char quickcheck = record->normalization_quick_check; @@ -772,7 +817,10 @@ &form, &PyUnicode_Type, &input)) return NULL; - if (PyUnicode_GetSize(input) == 0) { + if (PyUnicode_READY(input) == -1) + return NULL; + + if (PyUnicode_GET_LENGTH(input) == 0) { /* Special case empty input strings, since resizing them later would cause internal errors. */ Py_INCREF(input); -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Rename Py_BUILD_ASSERT to Py_BUILD_ASSERT_EXPR
by victor.stinner 29 Sep '11

29 Sep '11

http://hg.python.org/cpython/rev/60b76397080d changeset: 72524:60b76397080d user: Victor Stinner <vstinner(a)wyplay.com> date: Thu Sep 29 12:43:18 2011 +0200 summary: Rename Py_BUILD_ASSERT to Py_BUILD_ASSERT_EXPR To make it clearer that Py_BUILD_ASSERT_EXPR(cond) cannot be used as assert(cond). files: Include/pymacro.h | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -17,18 +17,17 @@ #define foo_to_char(foo) \ ((char *)(foo) \ - + Py_BUILD_ASSERT(offsetof(struct foo, string) == 0)) + + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT(cond) \ +#define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) - /* Get the number of elements in a visible array This does not work on pointers, or arrays declared as [], or function parameters. With correct compiler support, such usage will cause a build - error (see Py_BUILD_ASSERT). + error (see Py_BUILD_ASSERT_EXPR). Written by Rusty Russell, public domain, http://ccodearchive.net/ */ #if defined(__GNUC__) @@ -36,8 +35,8 @@ &a[0] degrades to a pointer: a different type from an array */ #define Py_ARRAY_LENGTH(array) \ (sizeof(array) / sizeof((array)[0]) \ - + Py_BUILD_ASSERT(!__builtin_types_compatible_p(typeof(array), \ - typeof(&(array)[0])))) + + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \ + typeof(&(array)[0])))) #else #define Py_ARRAY_LENGTH(array) \ (sizeof(array) / sizeof((array)[0])) -- Repository URL: http://hg.python.org/cpython

1 0