Mailman 3 September 2011 - Python-checkins

cpython: fill_number() ensures that the 'digits' string is ready
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/ad337be523bf changeset: 72496:ad337be523bf user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:50:42 2011 +0200 summary: fill_number() ensures that the 'digits' string is ready files: Python/formatter_unicode.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -587,7 +587,10 @@ /* Only for type 'c' special case, it has no digits. */ if (spec->n_digits != 0) { /* Fill the digits with InsertThousandsGrouping. */ - char *pdigits = PyUnicode_DATA(digits); + char *pdigits; + if (PyUnicode_READY(digits)) + return -1; + pdigits = PyUnicode_DATA(digits); if (PyUnicode_KIND(digits) < kind) { pdigits = _PyUnicode_AsKind(digits, kind); if (pdigits == NULL) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: fill_char() can now propagate an error
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/cf3557b65ae1 changeset: 72495:cf3557b65ae1 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:50:16 2011 +0200 summary: fill_char() can now propagate an error files: Python/formatter_unicode.c | 72 ++++++++++++++++--------- 1 files changed, 47 insertions(+), 25 deletions(-) diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -341,7 +341,7 @@ /* Pad on right. */ if (n_rpadding) - unicode_fill(s, start + nchars + n_lpadding, + unicode_fill(s, start + nchars + n_lpadding, start + nchars + n_lpadding + n_rpadding, fill_char); /* Pointer to the user content. */ @@ -501,7 +501,7 @@ spec->n_grouped_digits = 0; else spec->n_grouped_digits = _PyUnicode_InsertThousandsGrouping( - PyUnicode_1BYTE_KIND, NULL, 0, NULL, + PyUnicode_1BYTE_KIND, NULL, 0, NULL, spec->n_digits, spec->n_min_width, locale->grouping, locale->thousands_sep); @@ -541,11 +541,12 @@ /* Fill in the digit parts of a numbers's string representation, as determined in calc_number_widths(). - No error checking, since we know the buffer is the correct size. */ -static void + Return -1 on error, or 0 on success. */ +static int fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec, PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end, - PyObject *prefix, Py_ssize_t p_start, Py_UCS4 fill_char, + PyObject *prefix, Py_ssize_t p_start, + Py_UCS4 fill_char, LocaleInfo *locale, int toupper) { /* Used to keep track of digits, decimal, and remainder. */ @@ -589,11 +590,8 @@ char *pdigits = PyUnicode_DATA(digits); if (PyUnicode_KIND(digits) < kind) { pdigits = _PyUnicode_AsKind(digits, kind); - if (pdigits == NULL) { - /* XXX report exception */ - Py_FatalError("out of memory"); - return; - } + if (pdigits == NULL) + return -1; } #ifndef NDEBUG r = @@ -640,6 +638,7 @@ unicode_fill(out, pos, pos + spec->n_rpadding, fill_char); pos += spec->n_rpadding; } + return 0; } static char no_grouping[1] = {CHAR_MAX}; @@ -765,6 +764,7 @@ Py_ssize_t prefix; NumberFieldWidths spec; long x; + int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -886,10 +886,13 @@ goto done; /* Populate the memory. */ - fill_number(result, 0, &spec, tmp, inumeric_chars, inumeric_chars + n_digits, - tmp, prefix, - format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); + err = fill_number(result, 0, &spec, + tmp, inumeric_chars, inumeric_chars + n_digits, + tmp, prefix, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, format->type == 'X'); + if (err) + Py_CLEAR(result); done: Py_XDECREF(tmp); @@ -929,6 +932,7 @@ Py_UCS4 sign_char = '\0'; int float_type; /* Used to see if we have a nan, inf, or regular float. */ PyObject *unicode_tmp = NULL; + int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1010,7 +1014,7 @@ &locale); /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, 0, sign_char, unicode_tmp, index, + n_total = calc_number_widths(&spec, 0, sign_char, unicode_tmp, index, index + n_digits, n_remainder, has_decimal, &locale, format); @@ -1020,10 +1024,13 @@ goto done; /* Populate the memory. */ - fill_number(result, 0, &spec, unicode_tmp, index, index + n_digits, - NULL, 0, - format->fill_char == '\0' ? ' ' : format->fill_char, &locale, - 0); + err = fill_number(result, 0, &spec, + unicode_tmp, index, index + n_digits, + NULL, 0, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, 0); + if (err) + Py_CLEAR(result); done: PyMem_Free(buf); @@ -1077,6 +1084,7 @@ Py_ssize_t total; PyObject *re_unicode_tmp = NULL; PyObject *im_unicode_tmp = NULL; + int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1170,9 +1178,9 @@ /* Determine if we have any "remainder" (after the digits, might include decimal or exponent or both (or neither)) */ - parse_number(re_unicode_tmp, i_re, i_re + n_re_digits, + parse_number(re_unicode_tmp, i_re, i_re + n_re_digits, &n_re_remainder, &re_has_decimal); - parse_number(im_unicode_tmp, i_im, i_im + n_im_digits, + parse_number(im_unicode_tmp, i_im, i_im + n_im_digits, &n_im_remainder, &im_has_decimal); /* Determine the grouping, separator, and decimal point, if any. */ @@ -1225,12 +1233,26 @@ PyUnicode_WRITE(rkind, rdata, index++, '('); if (!skip_re) { - fill_number(result, index, &re_spec, re_unicode_tmp, - i_re, i_re + n_re_digits, NULL, 0, 0, &locale, 0); + err = fill_number(result, index, &re_spec, + re_unicode_tmp, i_re, i_re + n_re_digits, + NULL, 0, + 0, + &locale, 0); + if (err) { + Py_CLEAR(result); + goto done; + } index += n_re_total; } - fill_number(result, index, &im_spec, im_unicode_tmp, - i_im, i_im + n_im_digits, NULL, 0, 0, &locale, 0); + err = fill_number(result, index, &im_spec, + im_unicode_tmp, i_im, i_im + n_im_digits, + NULL, 0, + 0, + &locale, 0); + if (err) { + Py_CLEAR(result); + goto done; + } index += n_im_total; PyUnicode_WRITE(rkind, rdata, index++, 'j'); -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Check for PyUnicode_CopyCharacters() failure
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/c4dcb68e0711 changeset: 72493:c4dcb68e0711 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:39:17 2011 +0200 summary: Check for PyUnicode_CopyCharacters() failure files: Objects/unicodeobject.c | 120 ++++++++++++++++++--------- Python/compile.c | 10 +- 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1784,9 +1784,10 @@ (void) va_arg(vargs, char *); size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - PyUnicode_CopyCharacters((PyObject*)string, i, - *callresult, 0, - size); + if (PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, 0, + size) < 0) + goto fail; i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -1800,9 +1801,10 @@ Py_ssize_t size; assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); size = PyUnicode_GET_LENGTH(obj); - PyUnicode_CopyCharacters((PyObject*)string, i, - obj, 0, - size); + if (PyUnicode_CopyCharacters((PyObject*)string, i, + obj, 0, + size) < 0) + goto fail; i += size; break; } @@ -1814,17 +1816,19 @@ if (obj) { size = PyUnicode_GET_LENGTH(obj); assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); - PyUnicode_CopyCharacters((PyObject*)string, i, - obj, 0, - size); + if (PyUnicode_CopyCharacters((PyObject*)string, i, + obj, 0, + size) < 0) + goto fail; i += size; } else { size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - PyUnicode_CopyCharacters((PyObject*)string, i, - *callresult, - 0, size); + if (PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, + 0, size) < 0) + goto fail; i += size; Py_DECREF(*callresult); } @@ -1838,9 +1842,10 @@ /* unused, since we already have the result */ (void) va_arg(vargs, PyObject *); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - PyUnicode_CopyCharacters((PyObject*)string, i, - *callresult, 0, - PyUnicode_GET_LENGTH(*callresult)); + if (PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, 0, + PyUnicode_GET_LENGTH(*callresult)) < 0) + goto fail; i += PyUnicode_GET_LENGTH(*callresult); /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -8141,8 +8146,7 @@ else { /* In case the maximum character changed, we need to convert the string to the new category. */ - PyObject *v = PyUnicode_New( - PyUnicode_GET_LENGTH(self), maxchar_new); + PyObject *v = PyUnicode_New(PyUnicode_GET_LENGTH(self), maxchar_new); if (v == NULL) { Py_DECREF(u); return NULL; @@ -8151,12 +8155,25 @@ /* If the maxchar increased so that the kind changed, not all characters are representable anymore and we need to fix the string again. This only happens in very few cases. */ - PyUnicode_CopyCharacters(v, 0, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)); + if (PyUnicode_CopyCharacters(v, 0, + (PyObject*)self, 0, + PyUnicode_GET_LENGTH(self)) < 0) + { + Py_DECREF(u); + return NULL; + } maxchar_old = fixfct((PyUnicodeObject*)v); assert(maxchar_old > 0 && maxchar_old <= maxchar_new); } - else - PyUnicode_CopyCharacters(v, 0, u, 0, PyUnicode_GET_LENGTH(self)); + else { + if (PyUnicode_CopyCharacters(v, 0, + u, 0, + PyUnicode_GET_LENGTH(self)) < 0) + { + Py_DECREF(u); + return NULL; + } + } Py_DECREF(u); return v; @@ -8455,12 +8472,14 @@ itemlen = PyUnicode_GET_LENGTH(item); /* Copy item, and maybe the separator. */ if (i) { - PyUnicode_CopyCharacters(res, res_offset, - sep, 0, seplen); + if (PyUnicode_CopyCharacters(res, res_offset, + sep, 0, seplen) < 0) + goto onError; res_offset += seplen; } - PyUnicode_CopyCharacters(res, res_offset, - item, 0, itemlen); + if (PyUnicode_CopyCharacters(res, res_offset, + item, 0, itemlen) < 0) + goto onError; res_offset += itemlen; } assert(res_offset == PyUnicode_GET_LENGTH(res)); @@ -8508,6 +8527,8 @@ { PyObject *u; Py_UCS4 maxchar; + int kind; + void *data; if (left < 0) left = 0; @@ -8528,14 +8549,21 @@ if (fill > maxchar) maxchar = fill; u = PyUnicode_New(left + _PyUnicode_LENGTH(self) + right, maxchar); - if (u) { - int kind = PyUnicode_KIND(u); - void *data = PyUnicode_DATA(u); - if (left) - FILL(kind, data, fill, 0, left); - if (right) - FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); - PyUnicode_CopyCharacters(u, left, (PyObject*)self, 0, _PyUnicode_LENGTH(self)); + if (!u) + return NULL; + + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); + if (left) + FILL(kind, data, fill, 0, left); + if (right) + FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + if (PyUnicode_CopyCharacters(u, left, + (PyObject*)self, 0, + _PyUnicode_LENGTH(self)) < 0) + { + Py_DECREF(u); + return NULL; } return (PyUnicodeObject*)u; @@ -8821,8 +8849,12 @@ u = PyUnicode_New(slen, maxchar); if (!u) goto error; - PyUnicode_CopyCharacters(u, 0, - (PyObject*)self, 0, slen); + if (PyUnicode_CopyCharacters(u, 0, + (PyObject*)self, 0, slen) < 0) + { + Py_DECREF(u); + return NULL; + } rkind = PyUnicode_KIND(u); for (i = 0; i < PyUnicode_GET_LENGTH(u); i++) if (PyUnicode_READ(rkind, PyUnicode_DATA(u), i) == u1) { @@ -9437,8 +9469,7 @@ goto onError; maxchar = PyUnicode_MAX_CHAR_VALUE(u); - if (PyUnicode_MAX_CHAR_VALUE(v) > maxchar) - maxchar = PyUnicode_MAX_CHAR_VALUE(v); + maxchar = PY_MAX(maxchar, PyUnicode_MAX_CHAR_VALUE(v)); /* Concat the two Unicode strings */ w = PyUnicode_New( @@ -9446,9 +9477,12 @@ maxchar); if (w == NULL) goto onError; - PyUnicode_CopyCharacters(w, 0, u, 0, PyUnicode_GET_LENGTH(u)); - PyUnicode_CopyCharacters(w, PyUnicode_GET_LENGTH(u), v, 0, - PyUnicode_GET_LENGTH(v)); + if (PyUnicode_CopyCharacters(w, 0, u, 0, PyUnicode_GET_LENGTH(u)) < 0) + goto onError; + if (PyUnicode_CopyCharacters(w, PyUnicode_GET_LENGTH(u), + v, 0, + PyUnicode_GET_LENGTH(v)) < 0) + goto onError; Py_DECREF(u); Py_DECREF(v); return w; @@ -10396,8 +10430,12 @@ unicode = PyUnicode_New(len, maxchar); if (unicode == NULL) return NULL; - PyUnicode_CopyCharacters(unicode, 0, - (PyObject*)self, start, len); + if (PyUnicode_CopyCharacters(unicode, 0, + (PyObject*)self, start, len) < 0) + { + Py_DECREF(unicode); + return NULL; + } return unicode; } diff --git a/Python/compile.c b/Python/compile.c --- a/Python/compile.c +++ b/Python/compile.c @@ -245,8 +245,14 @@ return 0; /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_'); - PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen); - PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen); + if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) { + Py_DECREF(result); + return NULL; + } + if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) { + Py_DECREF(result); + return NULL; + } return result; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Strip trailing spaces in unicodeobject.[ch]
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/93cc591cffe4 changeset: 72494:93cc591cffe4 user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:41:31 2011 +0200 summary: Strip trailing spaces in unicodeobject.[ch] files: Include/unicodeobject.h | 10 +++++----- Objects/unicodeobject.c | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -356,13 +356,13 @@ #define PyUnicode_IS_COMPACT(op) \ (((PyASCIIObject*)(op))->state.compact) -/* Return one of the PyUnicode_*_KIND values defined above. */ +/* Return one of the PyUnicode_*_KIND values defined above. */ #define PyUnicode_KIND(op) \ (assert(PyUnicode_Check(op)), \ assert(PyUnicode_IS_READY(op)), \ ((PyASCIIObject *)(op))->state.kind) -/* Return a void pointer to the raw unicode buffer. */ +/* Return a void pointer to the raw unicode buffer. */ #define _PyUnicode_COMPACT_DATA(op) \ (PyUnicode_IS_COMPACT_ASCII(op) ? \ ((void*)((PyASCIIObject*)(op) + 1)) : \ @@ -509,7 +509,7 @@ /* Initializes the canonical string representation from a the deprected wstr/Py_UNICODE representation. This function is used to convert - unicode objects which were created using the old API to the new flexible + unicode objects which were created using the old API to the new flexible format introduced with PEP 393. The PyUnicode_READY() macro can be more efficient if the string is already ready. */ #ifndef Py_LIMITED_API @@ -641,7 +641,7 @@ PyObject *unicode ); -/* Get the number of Py_UNICODE units in the +/* Get the number of Py_UNICODE units in the string representation. */ PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize( @@ -857,7 +857,7 @@ In case of an error, no *size is set. - This funcation caches the UTF-8 encoded string in the unicodeobject + This funcation caches the UTF-8 encoded string in the unicodeobject and subsequent calls will return the same string. The memory is relased when the unicodeobject is deallocated. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -643,7 +643,7 @@ PyUnicode_KIND_SIZE(to_kind, how_many)); return how_many; } - + if (from_kind > to_kind) { /* slow path to check for character overflow */ const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); @@ -678,7 +678,7 @@ ); return how_many; } - else if (from_kind == PyUnicode_1BYTE_KIND + else if (from_kind == PyUnicode_1BYTE_KIND && to_kind == PyUnicode_4BYTE_KIND) { _PyUnicode_CONVERT_BYTES( @@ -703,7 +703,7 @@ PyErr_Format(PyExc_ValueError, "Cannot copy UCS%u characters " "into a string of UCS%u characters", - 1 << (from_kind - 1), + 1 << (from_kind - 1), 1 << (to_kind -1)); return -1; } @@ -8155,8 +8155,8 @@ /* If the maxchar increased so that the kind changed, not all characters are representable anymore and we need to fix the string again. This only happens in very few cases. */ - if (PyUnicode_CopyCharacters(v, 0, - (PyObject*)self, 0, + if (PyUnicode_CopyCharacters(v, 0, + (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)) < 0) { Py_DECREF(u); @@ -8166,8 +8166,8 @@ assert(maxchar_old > 0 && maxchar_old <= maxchar_new); } else { - if (PyUnicode_CopyCharacters(v, 0, - u, 0, + if (PyUnicode_CopyCharacters(v, 0, + u, 0, PyUnicode_GET_LENGTH(self)) < 0) { Py_DECREF(u); @@ -8558,8 +8558,8 @@ FILL(kind, data, fill, 0, left); if (right) FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); - if (PyUnicode_CopyCharacters(u, left, - (PyObject*)self, 0, + if (PyUnicode_CopyCharacters(u, left, + (PyObject*)self, 0, _PyUnicode_LENGTH(self)) < 0) { Py_DECREF(u); @@ -9479,7 +9479,7 @@ goto onError; if (PyUnicode_CopyCharacters(w, 0, u, 0, PyUnicode_GET_LENGTH(u)) < 0) goto onError; - if (PyUnicode_CopyCharacters(w, PyUnicode_GET_LENGTH(u), + if (PyUnicode_CopyCharacters(w, PyUnicode_GET_LENGTH(u), v, 0, PyUnicode_GET_LENGTH(v)) < 0) goto onError; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: PyUnicode_CopyCharacters() checks for buffer and character overflow
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/e69b7f21823f changeset: 72492:e69b7f21823f user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:37:03 2011 +0200 summary: PyUnicode_CopyCharacters() checks for buffer and character overflow It now returns the number of written characters on success. files: Include/unicodeobject.h | 18 ++- Objects/unicodeobject.c | 165 +++++++++++++-------------- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -519,10 +519,22 @@ #endif /* Copy character from one unicode object into another, this function performs - character conversion when nessesary and falls back to memcpy if possible. - Return -1 and raise an exception on error, return 0 on success. */ + character conversion when necessary and falls back to memcpy if possible. + + Fail if 'to' is smaller than how_many or smaller than len(from)-from_start, + or if kind(from[from_start:from_start+how_many]) > kind(to). + + Return the number of written character, or return -1 and raise an exception + on error. + + Pseudo-code: + + how_many = min(how_many, len(from) - from_start) + to[to_start:to_start+how_many] = from[from_start:from_start+how_many] + return how_many + */ #ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyUnicode_CopyCharacters( +PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( PyObject *to, Py_ssize_t to_start, PyObject *from, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -606,13 +606,13 @@ } #endif -int +Py_ssize_t PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) { - int from_kind; - int to_kind; + unsigned int from_kind; + unsigned int to_kind; assert(PyUnicode_Check(from)); assert(PyUnicode_Check(to)); @@ -622,94 +622,89 @@ if (PyUnicode_READY(to)) return -1; + how_many = PY_MIN(PyUnicode_GET_LENGTH(from), how_many); + if (to_start + how_many > PyUnicode_GET_LENGTH(to)) { + PyErr_Format(PyExc_ValueError, + "Cannot write %zi characters at %zi " + "in a string of %zi characters", + how_many, to_start, PyUnicode_GET_LENGTH(to)); + return -1; + } + from_kind = PyUnicode_KIND(from); to_kind = PyUnicode_KIND(to); if (from_kind == to_kind) { - const Py_ssize_t char_size = PyUnicode_CHARACTER_SIZE(to); - Py_MEMCPY(PyUnicode_1BYTE_DATA(to) + (to_start * char_size), - PyUnicode_1BYTE_DATA(from) + (from_start * char_size), - how_many * char_size); - return 0; - } - - switch (from_kind) { - case PyUnicode_1BYTE_KIND: - switch (to_kind) { - case PyUnicode_2BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - unsigned char, Py_UCS2, - PyUnicode_1BYTE_DATA(from) + from_start, - PyUnicode_1BYTE_DATA(from) + from_start + how_many, - PyUnicode_2BYTE_DATA(to) + to_start - ); + /* fast path */ + Py_MEMCPY((char*)PyUnicode_DATA(to) + + PyUnicode_KIND_SIZE(to_kind, to_start), + (char*)PyUnicode_DATA(from) + + PyUnicode_KIND_SIZE(from_kind, from_start), + PyUnicode_KIND_SIZE(to_kind, how_many)); + return how_many; + } + + if (from_kind > to_kind) { + /* slow path to check for character overflow */ + const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); + void *from_data = PyUnicode_DATA(from); + void *to_data = PyUnicode_DATA(to); + Py_UCS4 ch, maxchar; + Py_ssize_t i; + int overflow; + + maxchar = 0; + for (i=0; i < how_many; i++) { + ch = PyUnicode_READ(from_kind, from_data, from_start + i); + if (ch > maxchar) { + maxchar = ch; + if (maxchar > to_maxchar) { + overflow = 1; break; - case PyUnicode_4BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - unsigned char, Py_UCS4, - PyUnicode_1BYTE_DATA(from) + from_start, - PyUnicode_1BYTE_DATA(from) + from_start + how_many, - PyUnicode_4BYTE_DATA(to) + to_start - ); - break; - default: - goto invalid_state; - } - break; - case PyUnicode_2BYTE_KIND: - switch (to_kind) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - Py_UCS2, unsigned char, - PyUnicode_2BYTE_DATA(from) + from_start, - PyUnicode_2BYTE_DATA(from) + from_start + how_many, - PyUnicode_1BYTE_DATA(to) + to_start - ); - break; - case PyUnicode_4BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - Py_UCS2, Py_UCS4, - PyUnicode_2BYTE_DATA(from) + from_start, - PyUnicode_2BYTE_DATA(from) + from_start + how_many, - PyUnicode_4BYTE_DATA(to) + to_start - ); - break; - default: - goto invalid_state; - } - break; - case PyUnicode_4BYTE_KIND: - switch (to_kind) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - Py_UCS4, unsigned char, - PyUnicode_4BYTE_DATA(from) + from_start, - PyUnicode_4BYTE_DATA(from) + from_start + how_many, - PyUnicode_1BYTE_DATA(to) + to_start - ); - break; - case PyUnicode_2BYTE_KIND: - _PyUnicode_CONVERT_BYTES( - Py_UCS4, Py_UCS2, - PyUnicode_4BYTE_DATA(from) + from_start, - PyUnicode_4BYTE_DATA(from) + from_start + how_many, - PyUnicode_2BYTE_DATA(to) + to_start - ); - break; - default: - goto invalid_state; - } - break; - default: - goto invalid_state; - } - return 0; - -invalid_state: + } + } + PyUnicode_WRITE(to_kind, to_data, to_start + i, ch); + } + if (!overflow) + return how_many; + } + else if (from_kind == PyUnicode_1BYTE_KIND && to_kind == PyUnicode_2BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS2, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_2BYTE_DATA(to) + to_start + ); + return how_many; + } + else if (from_kind == PyUnicode_1BYTE_KIND + && to_kind == PyUnicode_4BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS4, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + return how_many; + } + else if (from_kind == PyUnicode_2BYTE_KIND + && to_kind == PyUnicode_4BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS4, + PyUnicode_2BYTE_DATA(from) + from_start, + PyUnicode_2BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + return how_many; + } PyErr_Format(PyExc_ValueError, - "Impossible kind state (from=%i, to=%i) " - "in PyUnicode_CopyCharacters", - from_kind, to_kind); + "Cannot copy UCS%u characters " + "into a string of UCS%u characters", + 1 << (from_kind - 1), + 1 << (to_kind -1)); return -1; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Mark PyUnicode_CONVERT_BYTES as private
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/256f7c55672f changeset: 72491:256f7c55672f user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 21:39:49 2011 +0200 summary: Mark PyUnicode_CONVERT_BYTES as private files: Include/unicodeobject.h | 15 ------------ Objects/unicodeobject.c | 35 ++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -468,21 +468,6 @@ (PyUnicode_IS_READY(op) ? \ 0 : _PyUnicode_Ready((PyUnicodeObject *)(op)))) -/* Generic helper macro to convert characters of different types. - from_type and to_type have to be valid type names, begin and end - are pointers to the source characters which should be of type - "from_type *". to is a pointer of type "to_type *" and points to the - buffer where the result characters are written to. */ -#define PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ - do { \ - const from_type *iter_; to_type *to_; \ - for (iter_ = (begin), to_ = (to_type *)(to); \ - iter_ < (end); \ - ++iter_, ++to_) { \ - *to_ = (to_type)*iter_; \ - } \ - } while (0) - /* Return a maximum character value which is suitable for creating another string based on op. This is always an approximation but more efficient than interating over the string. */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -90,6 +90,21 @@ extern "C" { #endif +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + const from_type *iter_; to_type *to_; \ + for (iter_ = (begin), to_ = (to_type *)(to); \ + iter_ < (end); \ + ++iter_, ++to_) { \ + *to_ = (to_type)*iter_; \ + } \ + } while (0) + #define _PyUnicode_WSTR(op) (((PyASCIIObject*)(op))->wstr) #define _PyUnicode_WSTR_LENGTH(op) (((PyCompactUnicodeObject*)(op))->wstr_length) #define _PyUnicode_LENGTH(op) (((PyASCIIObject *)(op))->length) @@ -622,7 +637,7 @@ case PyUnicode_1BYTE_KIND: switch (to_kind) { case PyUnicode_2BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( unsigned char, Py_UCS2, PyUnicode_1BYTE_DATA(from) + from_start, PyUnicode_1BYTE_DATA(from) + from_start + how_many, @@ -630,7 +645,7 @@ ); break; case PyUnicode_4BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( unsigned char, Py_UCS4, PyUnicode_1BYTE_DATA(from) + from_start, PyUnicode_1BYTE_DATA(from) + from_start + how_many, @@ -644,7 +659,7 @@ case PyUnicode_2BYTE_KIND: switch (to_kind) { case PyUnicode_1BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( Py_UCS2, unsigned char, PyUnicode_2BYTE_DATA(from) + from_start, PyUnicode_2BYTE_DATA(from) + from_start + how_many, @@ -652,7 +667,7 @@ ); break; case PyUnicode_4BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( Py_UCS2, Py_UCS4, PyUnicode_2BYTE_DATA(from) + from_start, PyUnicode_2BYTE_DATA(from) + from_start + how_many, @@ -666,7 +681,7 @@ case PyUnicode_4BYTE_KIND: switch (to_kind) { case PyUnicode_1BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( Py_UCS4, unsigned char, PyUnicode_4BYTE_DATA(from) + from_start, PyUnicode_4BYTE_DATA(from) + from_start + how_many, @@ -674,7 +689,7 @@ ); break; case PyUnicode_2BYTE_KIND: - PyUnicode_CONVERT_BYTES( + _PyUnicode_CONVERT_BYTES( Py_UCS4, Py_UCS2, PyUnicode_4BYTE_DATA(from) + from_start, PyUnicode_4BYTE_DATA(from) + from_start + how_many, @@ -792,7 +807,7 @@ PyErr_NoMemory(); return -1; } - PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, + _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, _PyUnicode_WSTR(unicode), end, PyUnicode_1BYTE_DATA(unicode)); PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; @@ -834,7 +849,7 @@ PyErr_NoMemory(); return -1; } - PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, + _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, _PyUnicode_WSTR(unicode), end, PyUnicode_2BYTE_DATA(unicode)); PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; @@ -1023,14 +1038,14 @@ switch (PyUnicode_KIND(unicode)) { case PyUnicode_1BYTE_KIND: - PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char, + _PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char, u, u + size, PyUnicode_1BYTE_DATA(unicode)); break; case PyUnicode_2BYTE_KIND: #if Py_UNICODE_SIZE == 2 Py_MEMCPY(PyUnicode_2BYTE_DATA(unicode), u, size * 2); #else - PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, + _PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, u, u + size, PyUnicode_2BYTE_DATA(unicode)); #endif break; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Add versionadded directive to new API function.
by georg.brandl Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/1b16230f1611 changeset: 72490:1b16230f1611 user: Georg Brandl <georg(a)python.org> date: Wed Sep 28 21:51:06 2011 +0200 summary: Add versionadded directive to new API function. files: Doc/c-api/unicode.rst | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1080,6 +1080,8 @@ first match; a value of ``-1`` indicates that no match was found, and ``-2`` indicates that an error occurred and an exception has been set. + .. versionadded:: 3.3 + .. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Rename new macros to conform to naming rules (function macros have "Py" prefix,
by georg.brandl Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/dacac31460c0 changeset: 72489:dacac31460c0 user: Georg Brandl <georg(a)python.org> date: Wed Sep 28 21:49:49 2011 +0200 summary: Rename new macros to conform to naming rules (function macros have "Py" prefix, not "PY"). files: Include/Python.h | 6 +++--- Modules/arraymodule.c | 2 +- Modules/zipimport.c | 8 ++++---- Objects/unicodeobject.c | 18 +++++++++--------- Python/import.c | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Include/Python.h b/Include/Python.h --- a/Include/Python.h +++ b/Include/Python.h @@ -160,9 +160,9 @@ #define PyDoc_STR(str) "" #endif -#define PY_ARRAY_LENGTH(array) (sizeof(array) / sizeof((array)[0])) +#define Py_ARRAY_LENGTH(array) (sizeof(array) / sizeof((array)[0])) -#define PY_MIN(x, y) (((x) > (y)) ? (y) : (x)) -#define PY_MAX(x, y) (((x) > (y)) ? (x) : (y)) +#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) +#define Py_MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif /* !Py_PYTHON_H */ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2810,7 +2810,7 @@ PyInit_array(void) { PyObject *m; - char buffer[PY_ARRAY_LENGTH(descriptors)], *p; + char buffer[Py_ARRAY_LENGTH(descriptors)], *p; PyObject *typecodes; Py_ssize_t size = 0; struct arraydescr *descr; diff --git a/Modules/zipimport.c b/Modules/zipimport.c --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -88,7 +88,7 @@ "archive path too long"); goto error; } - if (!PyUnicode_AsUCS4(pathobj, buf, PY_ARRAY_LENGTH(buf), 1)) + if (!PyUnicode_AsUCS4(pathobj, buf, Py_ARRAY_LENGTH(buf), 1)) goto error; #ifdef ALTSEP @@ -473,7 +473,7 @@ PyErr_SetString(ZipImportError, "path too long"); return NULL; } - if (!PyUnicode_AsUCS4(pathobj, buf, PY_ARRAY_LENGTH(buf), 1)) + if (!PyUnicode_AsUCS4(pathobj, buf, Py_ARRAY_LENGTH(buf), 1)) return NULL; path = buf; #ifdef ALTSEP @@ -484,7 +484,7 @@ #endif len = PyUnicode_GET_LENGTH(self->archive); if ((size_t)len < Py_UCS4_strlen(path)) { - if (!PyUnicode_AsUCS4(self->archive, archive, PY_ARRAY_LENGTH(archive), 1)) + if (!PyUnicode_AsUCS4(self->archive, archive, Py_ARRAY_LENGTH(archive), 1)) return NULL; if (Py_UCS4_strncmp(path, archive, len) == 0 && path[len] == SEP) { @@ -771,7 +771,7 @@ "Zip path name is too long"); return NULL; } - if (!PyUnicode_AsUCS4(archive, path, PY_ARRAY_LENGTH(path), 1)) + if (!PyUnicode_AsUCS4(archive, path, Py_ARRAY_LENGTH(path), 1)) return NULL; fp = _Py_fopen(archive, "rb"); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1521,7 +1521,7 @@ case 'c': { Py_UCS4 ordinal = va_arg(count, int); - maxchar = PY_MAX(maxchar, ordinal); + maxchar = Py_MAX(maxchar, ordinal); n++; break; } @@ -1617,7 +1617,7 @@ /* since PyUnicode_DecodeUTF8 returns already flexible unicode objects, there is no need to call ready on them */ argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; @@ -1630,7 +1630,7 @@ if (PyUnicode_READY(obj) == -1) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(obj); break; } @@ -1645,7 +1645,7 @@ if (PyUnicode_READY(obj) == -1) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(obj); *callresult++ = NULL; } @@ -1654,7 +1654,7 @@ if (!str_obj) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(str_obj); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(str_obj); *callresult++ = str_obj; } @@ -1669,7 +1669,7 @@ if (!str || PyUnicode_READY(str) == -1) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; @@ -1684,7 +1684,7 @@ if (!repr || PyUnicode_READY(repr) == -1) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(repr); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(repr); /* Remember the repr and switch to the next slot */ *callresult++ = repr; @@ -1699,7 +1699,7 @@ if (!ascii || PyUnicode_READY(ascii) == -1) goto fail; argmaxchar = PyUnicode_MAX_CHAR_VALUE(ascii); - maxchar = PY_MAX(maxchar, argmaxchar); + maxchar = Py_MAX(maxchar, argmaxchar); n += PyUnicode_GET_LENGTH(ascii); /* Remember the repr and switch to the next slot */ *callresult++ = ascii; @@ -11051,7 +11051,7 @@ kind1 = PyUnicode_KIND(str_in); kind2 = PyUnicode_KIND(sep_obj); - kind = PY_MAX(kind1, kind2); + kind = Py_MAX(kind1, kind2); buf1 = PyUnicode_DATA(str_in); if (kind1 != kind) buf1 = _PyUnicode_AsKind(str_in, kind); diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -1784,7 +1784,7 @@ return 0; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); PyErr_Clear(); return 0; @@ -1828,7 +1828,7 @@ #endif ) buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { PyErr_Clear(); return 0; } @@ -2787,7 +2787,7 @@ if (PyUnicode_READY(parent_name)) return NULL; buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Set Py_UNICODE_REPLACEMENT_CHARACTER type to Py_UCS4, instead of Py_UNICODE
by victor.stinner Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/17626ba912fb changeset: 72488:17626ba912fb user: Victor Stinner <victor.stinner(a)haypocalc.com> date: Wed Sep 28 20:29:27 2011 +0200 summary: Set Py_UNICODE_REPLACEMENT_CHARACTER type to Py_UCS4, instead of Py_UNICODE files: Include/unicodeobject.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -506,7 +506,7 @@ Unicode character U+FFFD is the official REPLACEMENT CHARACTER in Unicode 3.0. */ -#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD) +#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD) /* === Public API ========================================================= */ -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge default -> default): merge heads
by benjamin.peterson Sept. 28, 2011

Sept. 28, 2011

http://hg.python.org/cpython/rev/5a12416890c0 changeset: 72487:5a12416890c0 parent: 72486:347c950fb663 parent: 72485:83f43b58c988 user: Benjamin Peterson <benjamin(a)python.org> date: Wed Sep 28 10:48:40 2011 -0400 summary: merge heads files: Lib/argparse.py | 2 +- Lib/codecs.py | 14 +++++++------- Lib/collections/__init__.py | 2 +- Lib/difflib.py | 20 ++++++++++---------- Lib/doctest.py | 6 +++--- Lib/lib2to3/refactor.py | 4 ++-- Lib/string.py | 2 +- Lib/test/test_calendar.py | 2 +- Lib/test/test_email/__init__.py | 2 +- Lib/test/test_gzip.py | 4 ++-- Lib/test/test_inspect.py | 2 +- Lib/test/test_io.py | 4 ++-- Lib/test/test_nntplib.py | 4 ++-- Lib/test/test_pydoc.py | 4 ++-- Lib/test/test_tokenize.py | 2 +- Lib/unittest/case.py | 4 ++-- 16 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -641,7 +641,7 @@ """ def _fill_text(self, text, width, indent): - return ''.join([indent + line for line in text.splitlines(True)]) + return ''.join(indent + line for line in text.splitlines(keepends=True)) class RawTextHelpFormatter(RawDescriptionHelpFormatter): diff --git a/Lib/codecs.py b/Lib/codecs.py --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -484,7 +484,7 @@ if firstline: newchars, decodedbytes = \ self.decode(data[:exc.start], self.errors) - lines = newchars.splitlines(True) + lines = newchars.splitlines(keepends=True) if len(lines)<=1: raise else: @@ -526,7 +526,7 @@ self.charbuffer = self.linebuffer[0] self.linebuffer = None if not keepends: - line = line.splitlines(False)[0] + line = line.splitlines(keepends=False)[0] return line readsize = size or 72 @@ -543,7 +543,7 @@ data += self.read(size=1, chars=1) line += data - lines = line.splitlines(True) + lines = line.splitlines(keepends=True) if lines: if len(lines) > 1: # More than one line result; the first line is a full line @@ -559,10 +559,10 @@ # only one remaining line, put it back into charbuffer self.charbuffer = lines[0] + self.charbuffer if not keepends: - line = line.splitlines(False)[0] + line = line.splitlines(keepends=False)[0] break line0withend = lines[0] - line0withoutend = lines[0].splitlines(False)[0] + line0withoutend = lines[0].splitlines(keepends=False)[0] if line0withend != line0withoutend: # We really have a line end # Put the rest back together and keep it until the next call self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \ @@ -575,7 +575,7 @@ # we didn't get anything or this was our only try if not data or size is not None: if line and not keepends: - line = line.splitlines(False)[0] + line = line.splitlines(keepends=False)[0] break if readsize < 8000: readsize *= 2 @@ -803,7 +803,7 @@ data = self.reader.read() data, bytesencoded = self.encode(data, self.errors) - return data.splitlines(1) + return data.splitlines(keepends=True) def __next__(self): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -1045,7 +1045,7 @@ return self.data.split(sep, maxsplit) def rsplit(self, sep=None, maxsplit=-1): return self.data.rsplit(sep, maxsplit) - def splitlines(self, keepends=0): return self.data.splitlines(keepends) + def splitlines(self, keepends=False): return self.data.splitlines(keepends) def startswith(self, prefix, start=0, end=_sys.maxsize): return self.data.startswith(prefix, start, end) def strip(self, chars=None): return self.__class__(self.data.strip(chars)) diff --git a/Lib/difflib.py b/Lib/difflib.py --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -800,7 +800,7 @@ ... 2. Explicit is better than implicit. ... 3. Simple is better than complex. ... 4. Complex is better than complicated. - ... '''.splitlines(1) + ... '''.splitlines(keepends=True) >>> len(text1) 4 >>> text1[0][-1] @@ -809,7 +809,7 @@ ... 3. Simple is better than complex. ... 4. Complicated is better than complex. ... 5. Flat is better than nested. - ... '''.splitlines(1) + ... '''.splitlines(keepends=True) Next we instantiate a Differ object: @@ -896,8 +896,8 @@ Example: - >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(1), - ... 'ore\ntree\nemu\n'.splitlines(1))), + >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(True), + ... 'ore\ntree\nemu\n'.splitlines(True))), ... end="") - one ? ^ @@ -1269,8 +1269,8 @@ Example: - >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(1), - ... 'zero\none\ntree\nfour\n'.splitlines(1), 'Original', 'Current')), + >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), + ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')), ... end="") *** Original --- Current @@ -1339,8 +1339,8 @@ Example: - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), - ... 'ore\ntree\nemu\n'.splitlines(1)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> print(''.join(diff), end="") - one ? ^ @@ -2034,8 +2034,8 @@ Examples: - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), - ... 'ore\ntree\nemu\n'.splitlines(1)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> diff = list(diff) >>> print(''.join(restore(diff, 1)), end="") one diff --git a/Lib/doctest.py b/Lib/doctest.py --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1332,7 +1332,7 @@ m = self.__LINECACHE_FILENAME_RE.match(filename) if m and m.group('name') == self.test.name: example = self.test.examples[int(m.group('examplenum'))] - return example.source.splitlines(True) + return example.source.splitlines(keepends=True) else: return self.save_linecache_getlines(filename, module_globals) @@ -1595,8 +1595,8 @@ # Check if we should use diff. if self._do_a_fancy_diff(want, got, optionflags): # Split want & got into lines. - want_lines = want.splitlines(True) # True == keep line ends - got_lines = got.splitlines(True) + want_lines = want.splitlines(keepends=True) + got_lines = got.splitlines(keepends=True) # Use difflib to find their differences. if optionflags & REPORT_UDIFF: diff = difflib.unified_diff(want_lines, got_lines, n=2) diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -560,7 +560,7 @@ block_lineno = None indent = None lineno = 0 - for line in input.splitlines(True): + for line in input.splitlines(keepends=True): lineno += 1 if line.lstrip().startswith(self.PS1): if block is not None: @@ -604,7 +604,7 @@ filename, lineno, err.__class__.__name__, err) return block if self.refactor_tree(tree, filename): - new = str(tree).splitlines(True) + new = str(tree).splitlines(keepends=True) # Undo the adjustment of the line numbers in wrap_toks() below. clipped, new = new[:lineno-1], new[lineno-1:] assert clipped == ["\n"] * (lineno-1), clipped diff --git a/Lib/string.py b/Lib/string.py --- a/Lib/string.py +++ b/Lib/string.py @@ -84,7 +84,7 @@ def _invalid(self, mo): i = mo.start('invalid') - lines = self.template[:i].splitlines(True) + lines = self.template[:i].splitlines(keepends=True) if not lines: colno = 1 lineno = 1 diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -177,7 +177,7 @@ return not c.isspace() and not c.isdigit() lines = [] - for line in s.splitlines(False): + for line in s.splitlines(keepends=False): # Drop texts, as they are locale dependent if line and not filter(neitherspacenordigit, line): lines.append(line) diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py --- a/Lib/test/test_email/__init__.py +++ b/Lib/test/test_email/__init__.py @@ -38,7 +38,7 @@ return email.message_from_file(fp) def _bytes_repr(self, b): - return [repr(x) for x in b.splitlines(True)] + return [repr(x) for x in b.splitlines(keepends=True)] def assertBytesEqual(self, first, second, msg): """Our byte strings are really encoded strings; improve diff output""" diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -139,7 +139,7 @@ with io.BufferedReader(f) as r: lines = [line for line in r] - self.assertEqual(lines, 50 * data1.splitlines(True)) + self.assertEqual(lines, 50 * data1.splitlines(keepends=True)) def test_readline(self): self.test_write() @@ -340,7 +340,7 @@ def test_textio_readlines(self): # Issue #10791: TextIOWrapper.readlines() fails when wrapping GzipFile. - lines = (data1 * 50).decode("ascii").splitlines(True) + lines = (data1 * 50).decode("ascii").splitlines(keepends=True) self.test_write() with gzip.GzipFile(self.filename, 'r') as f: with io.TextIOWrapper(f, encoding="ascii") as t: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -304,7 +304,7 @@ getlines = linecache.getlines def monkey(filename, module_globals=None): if filename == fn: - return source.splitlines(True) + return source.splitlines(keepends=True) else: return getlines(filename, module_globals) linecache.getlines = monkey diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1935,8 +1935,8 @@ testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") for newline, expected in [ - (None, normalized.decode("ascii").splitlines(True)), - ("", testdata.decode("ascii").splitlines(True)), + (None, normalized.decode("ascii").splitlines(keepends=True)), + ("", testdata.decode("ascii").splitlines(keepends=True)), ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -1033,12 +1033,12 @@ self.assertEqual(resp, success_resp) # With an iterable of terminated lines def iterlines(b): - return iter(b.splitlines(True)) + return iter(b.splitlines(keepends=True)) resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines) self.assertEqual(resp, success_resp) # With an iterable of non-terminated lines def iterlines(b): - return iter(b.splitlines(False)) + return iter(b.splitlines(keepends=False)) resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines) self.assertEqual(resp, success_resp) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -238,8 +238,8 @@ def print_diffs(text1, text2): "Prints unified diffs for two texts" # XXX now obsolete, use unittest built-in support - lines1 = text1.splitlines(True) - lines2 = text2.splitlines(True) + lines1 = text1.splitlines(keepends=True) + lines2 = text2.splitlines(keepends=True) diffs = difflib.unified_diff(lines1, lines2, n=0, fromfile='expected', tofile='got') print('\n' + ''.join(diffs)) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -600,7 +600,7 @@ f.close() tokens1 = [tok[:2] for tok in token_list] new_bytes = untokenize(tokens1) - readline = (line for line in new_bytes.splitlines(1)).__next__ + readline = (line for line in new_bytes.splitlines(keepends=True)).__next__ tokens2 = [tok[:2] for tok in tokenize(readline)] return tokens1 == tokens2 diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1010,8 +1010,8 @@ if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) - firstlines = first.splitlines(True) - secondlines = second.splitlines(True) + firstlines = first.splitlines(keepends=True) + secondlines = second.splitlines(keepends=True) if len(firstlines) == 1 and first.strip('\r\n') == first: firstlines = [first + '\n'] secondlines = [second + '\n'] -- Repository URL: http://hg.python.org/cpython

1 0