[Python-checkins] cpython: Speedup str[a:b] and PyUnicode_FromKindAndData

[victor.stinner]

http://hg.python.org/cpython/rev/61ca5262b6c4
changeset:   72685:61ca5262b6c4
user:        Victor Stinner <vstinner at wyplay.com>
date:        Wed Oct 05 14:01:42 2011 +0200
summary:
  Speedup str[a:b] and PyUnicode_FromKindAndData

 * str[a:b] doesn't scan the string for the maximum character if the string
   is ascii only
 * PyUnicode_FromKindAndData() stops if we are sure that we cannot use a
   shorter character type. For example, _PyUnicode_FromUCS1() stops if we
   have at least one character in range U+0080-U+00FF

files:
  Include/unicodeobject.h |   2 +
  Objects/unicodeobject.c |  78 ++++++++++++++++++----------
  2 files changed, 52 insertions(+), 28 deletions(-)


diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -654,6 +654,8 @@
     const char *u              /* UTF-8 encoded string */
     );
 
+/* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters.
+   Scan the string to find the maximum character. */
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData(
     int kind,
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -969,7 +969,7 @@
 
     if (from_kind == to_kind
         /* deny latin1 => ascii */
-        && PyUnicode_MAX_CHAR_VALUE(to) >= PyUnicode_MAX_CHAR_VALUE(from))
+        && !(!PyUnicode_IS_ASCII(from) && PyUnicode_IS_ASCII(to)))
     {
         Py_MEMCPY((char*)to_data
                       + PyUnicode_KIND_SIZE(to_kind, to_start),
@@ -1013,9 +1013,7 @@
         /* check if max_char(from substring) <= max_char(to) */
         if (from_kind > to_kind
                 /* latin1 => ascii */
-            || (PyUnicode_IS_ASCII(to)
-                && to_kind == PyUnicode_1BYTE_KIND
-                && !PyUnicode_IS_ASCII(from)))
+            || (!PyUnicode_IS_ASCII(from) && PyUnicode_IS_ASCII(to)))
         {
             /* slow path to check for character overflow */
             const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to);
@@ -1528,15 +1526,17 @@
 _PyUnicode_FromUCS1(const unsigned char* u, Py_ssize_t size)
 {
     PyObject *res;
-    unsigned char max = 127;
+    unsigned char max_char = 127;
     Py_ssize_t i;
+
+    assert(size >= 0);
     for (i = 0; i < size; i++) {
         if (u[i] & 0x80) {
-            max = 255;
+            max_char = 255;
             break;
         }
     }
-    res = PyUnicode_New(size, max);
+    res = PyUnicode_New(size, max_char);
     if (!res)
         return NULL;
     memcpy(PyUnicode_1BYTE_DATA(res), u, size);
@@ -1547,15 +1547,21 @@
 _PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size)
 {
     PyObject *res;
-    Py_UCS2 max = 0;
+    Py_UCS2 max_char = 0;
     Py_ssize_t i;
-    for (i = 0; i < size; i++)
-        if (u[i] > max)
-            max = u[i];
-    res = PyUnicode_New(size, max);
+
+    assert(size >= 0);
+    for (i = 0; i < size; i++) {
+        if (u[i] > max_char) {
+            max_char = u[i];
+            if (max_char >= 256)
+                break;
+        }
+    }
+    res = PyUnicode_New(size, max_char);
     if (!res)
         return NULL;
-    if (max >= 256)
+    if (max_char >= 256)
         memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size);
     else
         for (i = 0; i < size; i++)
@@ -1567,15 +1573,21 @@
 _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
 {
     PyObject *res;
-    Py_UCS4 max = 0;
+    Py_UCS4 max_char = 0;
     Py_ssize_t i;
-    for (i = 0; i < size; i++)
-        if (u[i] > max)
-            max = u[i];
-    res = PyUnicode_New(size, max);
+
+    assert(size >= 0);
+    for (i = 0; i < size; i++) {
+        if (u[i] > max_char) {
+            max_char = u[i];
+            if (max_char >= 0x10000)
+                break;
+        }
+    }
+    res = PyUnicode_New(size, max_char);
     if (!res)
         return NULL;
-    if (max >= 0x10000)
+    if (max_char >= 0x10000)
         memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size);
     else {
         int kind = PyUnicode_KIND(res);
@@ -1596,9 +1608,11 @@
         return _PyUnicode_FromUCS2(buffer, size);
     case PyUnicode_4BYTE_KIND:
         return _PyUnicode_FromUCS4(buffer, size);
-    }
-    PyErr_SetString(PyExc_SystemError, "invalid kind");
-    return NULL;
+    default:
+        assert(0 && "invalid kind");
+        PyErr_SetString(PyExc_SystemError, "invalid kind");
+        return NULL;
+    }
 }
 
 PyObject*
@@ -9383,11 +9397,12 @@
             maxchar = PyUnicode_MAX_CHAR_VALUE(self);
             /* Replacing u1 with u2 may cause a maxchar reduction in the
                result string. */
-            mayshrink = maxchar > 127;
             if (u2 > maxchar) {
                 maxchar = u2;
                 mayshrink = 0;
             }
+            else
+                mayshrink = maxchar > 127;
             u = PyUnicode_New(slen, maxchar);
             if (!u)
                 goto error;
@@ -11039,11 +11054,18 @@
         return NULL;
     }
 
-    kind = PyUnicode_KIND(self);
-    data = PyUnicode_1BYTE_DATA(self);
-    return PyUnicode_FromKindAndData(kind,
-                                     data + PyUnicode_KIND_SIZE(kind, start),
-                                     length);
+    if (PyUnicode_IS_ASCII(self)) {
+        kind = PyUnicode_KIND(self);
+        data = PyUnicode_1BYTE_DATA(self);
+        return unicode_fromascii(data + start, length);
+    }
+    else {
+        kind = PyUnicode_KIND(self);
+        data = PyUnicode_1BYTE_DATA(self);
+        return PyUnicode_FromKindAndData(kind,
+                                         data + PyUnicode_KIND_SIZE(kind, start),
+                                         length);
+    }
 }
 
 static PyObject *

-- 
Repository URL: http://hg.python.org/cpython