[Python-3000-checkins] r55282 - python/branches/py3k-struni/Objects/unicodeobject.c
walter.doerwald
python-3000-checkins at python.org
Sat May 12 13:08:09 CEST 2007
Author: walter.doerwald
Date: Sat May 12 13:08:06 2007
New Revision: 55282
Modified:
python/branches/py3k-struni/Objects/unicodeobject.c
Log:
Change PyUnicode_EncodeUnicodeEscape() to return a bytes object.
However PyUnicode_AsUnicodeEscapeString() (which is used by
Objects/fileobject.c::file_repr()) still returns a str8 object.
Give unicode_repr() it's own implementation which returns a str8
object (it was formerly just calling unicodeescape_string()
which was used to implement PyUnicode_EncodeUnicodeEscape() too),
because once repr() is required to return unicode objects it
needs its own implementation anyway.
Modified: python/branches/py3k-struni/Objects/unicodeobject.c
==============================================================================
--- python/branches/py3k-struni/Objects/unicodeobject.c (original)
+++ python/branches/py3k-struni/Objects/unicodeobject.c Sat May 12 13:08:06 2007
@@ -2094,16 +2094,14 @@
return NULL;
}
-static
-PyObject *unicodeescape_string(const Py_UNICODE *s,
- Py_ssize_t size,
- int quotes)
+static const char *hexdigits = "0123456789abcdef";
+
+PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
+ Py_ssize_t size)
{
PyObject *repr;
char *p;
- static const char *hexdigit = "0123456789abcdef";
-
/* XXX(nnorwitz): rather than over-allocating, it would be
better to choose a different scheme. Perhaps scan the
first N-chars of the string and allocate based on that size.
@@ -2122,8 +2120,7 @@
escape.
*/
- repr = PyString_FromStringAndSize(NULL,
- 2
+ repr = PyBytes_FromStringAndSize(NULL,
#ifdef Py_UNICODE_WIDE
+ 10*size
#else
@@ -2133,21 +2130,16 @@
if (repr == NULL)
return NULL;
- p = PyString_AS_STRING(repr);
+ p = PyBytes_AS_STRING(repr);
- if (quotes) {
- *p++ = (findchar(s, size, '\'') &&
- !findchar(s, size, '"')) ? '"' : '\'';
- }
while (size-- > 0) {
Py_UNICODE ch = *s++;
- /* Escape quotes and backslashes */
- if ((quotes &&
- ch == (Py_UNICODE) PyString_AS_STRING(repr)[0]) || ch == '\\') {
+ /* Escape backslashes */
+ if (ch == '\\') {
*p++ = '\\';
*p++ = (char) ch;
- continue;
+ continue;
}
#ifdef Py_UNICODE_WIDE
@@ -2155,14 +2147,14 @@
else if (ch >= 0x10000) {
*p++ = '\\';
*p++ = 'U';
- *p++ = hexdigit[(ch >> 28) & 0x0000000F];
- *p++ = hexdigit[(ch >> 24) & 0x0000000F];
- *p++ = hexdigit[(ch >> 20) & 0x0000000F];
- *p++ = hexdigit[(ch >> 16) & 0x0000000F];
- *p++ = hexdigit[(ch >> 12) & 0x0000000F];
- *p++ = hexdigit[(ch >> 8) & 0x0000000F];
- *p++ = hexdigit[(ch >> 4) & 0x0000000F];
- *p++ = hexdigit[ch & 0x0000000F];
+ *p++ = hexdigits[(ch >> 28) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 24) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 20) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 16) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 12) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 8) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 4) & 0x0000000F];
+ *p++ = hexdigits[ch & 0x0000000F];
continue;
}
#else
@@ -2177,14 +2169,14 @@
ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
*p++ = '\\';
*p++ = 'U';
- *p++ = hexdigit[(ucs >> 28) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 24) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 20) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 16) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 12) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 8) & 0x0000000F];
- *p++ = hexdigit[(ucs >> 4) & 0x0000000F];
- *p++ = hexdigit[ucs & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 28) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 24) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 20) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 16) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 12) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 8) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 4) & 0x0000000F];
+ *p++ = hexdigits[ucs & 0x0000000F];
continue;
}
/* Fall through: isolated surrogates are copied as-is */
@@ -2197,10 +2189,10 @@
if (ch >= 256) {
*p++ = '\\';
*p++ = 'u';
- *p++ = hexdigit[(ch >> 12) & 0x000F];
- *p++ = hexdigit[(ch >> 8) & 0x000F];
- *p++ = hexdigit[(ch >> 4) & 0x000F];
- *p++ = hexdigit[ch & 0x000F];
+ *p++ = hexdigits[(ch >> 12) & 0x000F];
+ *p++ = hexdigits[(ch >> 8) & 0x000F];
+ *p++ = hexdigits[(ch >> 4) & 0x000F];
+ *p++ = hexdigits[ch & 0x000F];
}
/* Map special whitespace to '\t', \n', '\r' */
@@ -2221,36 +2213,39 @@
else if (ch < ' ' || ch >= 0x7F) {
*p++ = '\\';
*p++ = 'x';
- *p++ = hexdigit[(ch >> 4) & 0x000F];
- *p++ = hexdigit[ch & 0x000F];
+ *p++ = hexdigits[(ch >> 4) & 0x000F];
+ *p++ = hexdigits[ch & 0x000F];
}
/* Copy everything else as-is */
else
*p++ = (char) ch;
}
- if (quotes)
- *p++ = PyString_AS_STRING(repr)[0];
*p = '\0';
- _PyString_Resize(&repr, p - PyString_AS_STRING(repr));
+ if (PyBytes_Resize(repr, p - PyBytes_AS_STRING(repr))) {
+ Py_DECREF(repr);
+ return NULL;
+ }
return repr;
}
-PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
- Py_ssize_t size)
-{
- return unicodeescape_string(s, size, 0);
-}
-
PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
{
+ PyObject *s, *result;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
- return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
- PyUnicode_GET_SIZE(unicode));
+ s = PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
+ PyUnicode_GET_SIZE(unicode));
+
+ if (!s)
+ return NULL;
+ result = PyString_FromStringAndSize(PyBytes_AS_STRING(s),
+ PyBytes_GET_SIZE(s));
+ Py_DECREF(s);
+ return result;
}
/* --- Raw Unicode Escape Codec ------------------------------------------- */
@@ -6521,9 +6516,140 @@
static
PyObject *unicode_repr(PyObject *unicode)
{
- return unicodeescape_string(PyUnicode_AS_UNICODE(unicode),
- PyUnicode_GET_SIZE(unicode),
- 1);
+ PyObject *repr;
+ char *p;
+ Py_UNICODE *s = PyUnicode_AS_UNICODE(unicode);
+ Py_ssize_t size = PyUnicode_GET_SIZE(unicode);
+
+ /* XXX(nnorwitz): rather than over-allocating, it would be
+ better to choose a different scheme. Perhaps scan the
+ first N-chars of the string and allocate based on that size.
+ */
+ /* Initial allocation is based on the longest-possible unichr
+ escape.
+
+ In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source
+ unichr, so in this case it's the longest unichr escape. In
+ narrow (UTF-16) builds this is five chars per source unichr
+ since there are two unichrs in the surrogate pair, so in narrow
+ (UTF-16) builds it's not the longest unichr escape.
+
+ In wide or narrow builds '\uxxxx' is 6 chars per source unichr,
+ so in the narrow (UTF-16) build case it's the longest unichr
+ escape.
+ */
+
+ repr = PyString_FromStringAndSize(NULL,
+ 2 /* quotes */
+#ifdef Py_UNICODE_WIDE
+ + 10*size
+#else
+ + 6*size
+#endif
+ + 1);
+ if (repr == NULL)
+ return NULL;
+
+ p = PyString_AS_STRING(repr);
+
+ /* Add quote */
+ *p++ = (findchar(s, size, '\'') &&
+ !findchar(s, size, '"')) ? '"' : '\'';
+ while (size-- > 0) {
+ Py_UNICODE ch = *s++;
+
+ /* Escape quotes and backslashes */
+ if ((ch == (Py_UNICODE) PyString_AS_STRING(repr)[0]) || (ch == '\\')) {
+ *p++ = '\\';
+ *p++ = (char) ch;
+ continue;
+ }
+
+#ifdef Py_UNICODE_WIDE
+ /* Map 21-bit characters to '\U00xxxxxx' */
+ else if (ch >= 0x10000) {
+ *p++ = '\\';
+ *p++ = 'U';
+ *p++ = hexdigits[(ch >> 28) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 24) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 20) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 16) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 12) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 8) & 0x0000000F];
+ *p++ = hexdigits[(ch >> 4) & 0x0000000F];
+ *p++ = hexdigits[ch & 0x0000000F];
+ continue;
+ }
+#else
+ /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
+ else if (ch >= 0xD800 && ch < 0xDC00) {
+ Py_UNICODE ch2;
+ Py_UCS4 ucs;
+
+ ch2 = *s++;
+ size--;
+ if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
+ ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
+ *p++ = '\\';
+ *p++ = 'U';
+ *p++ = hexdigits[(ucs >> 28) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 24) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 20) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 16) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 12) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 8) & 0x0000000F];
+ *p++ = hexdigits[(ucs >> 4) & 0x0000000F];
+ *p++ = hexdigits[ucs & 0x0000000F];
+ continue;
+ }
+ /* Fall through: isolated surrogates are copied as-is */
+ s--;
+ size++;
+ }
+#endif
+
+ /* Map 16-bit characters to '\uxxxx' */
+ if (ch >= 256) {
+ *p++ = '\\';
+ *p++ = 'u';
+ *p++ = hexdigits[(ch >> 12) & 0x000F];
+ *p++ = hexdigits[(ch >> 8) & 0x000F];
+ *p++ = hexdigits[(ch >> 4) & 0x000F];
+ *p++ = hexdigits[ch & 0x000F];
+ }
+
+ /* Map special whitespace to '\t', \n', '\r' */
+ else if (ch == '\t') {
+ *p++ = '\\';
+ *p++ = 't';
+ }
+ else if (ch == '\n') {
+ *p++ = '\\';
+ *p++ = 'n';
+ }
+ else if (ch == '\r') {
+ *p++ = '\\';
+ *p++ = 'r';
+ }
+
+ /* Map non-printable US ASCII to '\xhh' */
+ else if (ch < ' ' || ch >= 0x7F) {
+ *p++ = '\\';
+ *p++ = 'x';
+ *p++ = hexdigits[(ch >> 4) & 0x000F];
+ *p++ = hexdigits[ch & 0x000F];
+ }
+
+ /* Copy everything else as-is */
+ else
+ *p++ = (char) ch;
+ }
+ /* Add quote */
+ *p++ = PyString_AS_STRING(repr)[0];
+
+ *p = '\0';
+ _PyString_Resize(&repr, p - PyString_AS_STRING(repr));
+ return repr;
}
PyDoc_STRVAR(rfind__doc__,
More information about the Python-3000-checkins
mailing list