Attached you find the latest update of the Unicode implementation. The patch is against the current CVS version. It includes the fix I posted yesterday for the core dump problem in codecs.c (was introduced by my previous patch set -- sorry), adds more tests for the codecs and two new parser markers "es" and "es#". -- Marc-Andre Lemburg ______________________________________________________________________ Business: http://www.lemburg.com/ Python Pages: http://www.lemburg.com/python/ Only in CVS-Python/Doc/tools: anno-api.py diff -u -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS CVS-Python/Lib/codecs.py Python+Unicode/Lib/codecs.py --- CVS-Python/Lib/codecs.py Thu Mar 23 23:58:41 2000 +++ Python+Unicode/Lib/codecs.py Fri Mar 17 23:51:01 2000 @@ -46,7 +46,7 @@ handling schemes by providing the errors argument. These string values are defined: - 'strict' - raise an error (or a subclass) + 'strict' - raise a ValueError error (or a subclass) 'ignore' - ignore the character and continue with the next 'replace' - replace with a suitable replacement character; Python will use the official U+FFFD REPLACEMENT diff -u -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS CVS-Python/Lib/test/output/test_unicode Python+Unicode/Lib/test/output/test_unicode --- CVS-Python/Lib/test/output/test_unicode Fri Mar 24 22:21:26 2000 +++ Python+Unicode/Lib/test/output/test_unicode Sat Mar 11 00:23:21 2000 @@ -1,5 +1,4 @@ test_unicode Testing Unicode comparisons... done. -Testing Unicode contains method... done. Testing Unicode formatting strings... done. Testing unicodedata module... done. diff -u -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS CVS-Python/Lib/test/test_unicode.py Python+Unicode/Lib/test/test_unicode.py --- CVS-Python/Lib/test/test_unicode.py Thu Mar 23 23:58:47 2000 +++ Python+Unicode/Lib/test/test_unicode.py Fri Mar 24 00:29:43 2000 @@ -293,3 +293,33 @@ assert unicodedata.combining(u'\u20e1') == 230 print 'done.' + +# Test builtin codecs +print 'Testing builtin codecs...', + +assert unicode('hello','ascii') == u'hello' +assert unicode('hello','utf-8') == u'hello' +assert unicode('hello','utf8') == u'hello' +assert unicode('hello','latin-1') == u'hello' + +assert u'hello'.encode('ascii') == 'hello' +assert u'hello'.encode('utf-8') == 'hello' +assert u'hello'.encode('utf8') == 'hello' +assert u'hello'.encode('utf-16-le') == 'h\000e\000l\000l\000o\000' +assert u'hello'.encode('utf-16-be') == '\000h\000e\000l\000l\000o' +assert u'hello'.encode('latin-1') == 'hello' + +u = u''.join(map(unichr, range(1024))) +for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', + 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): + assert unicode(u.encode(encoding),encoding) == u + +u = u''.join(map(unichr, range(256))) +for encoding in ('latin-1',): + assert unicode(u.encode(encoding),encoding) == u + +u = u''.join(map(unichr, range(128))) +for encoding in ('ascii',): + assert unicode(u.encode(encoding),encoding) == u + +print 'done.' diff -u -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS CVS-Python/Misc/unicode.txt Python+Unicode/Misc/unicode.txt --- CVS-Python/Misc/unicode.txt Thu Mar 23 23:58:48 2000 +++ Python+Unicode/Misc/unicode.txt Fri Mar 24 22:29:35 2000 @@ -715,21 +715,126 @@ These markers are used by the PyArg_ParseTuple() APIs: - 'U': Check for Unicode object and return a pointer to it + "U": Check for Unicode object and return a pointer to it - 's': For Unicode objects: auto convert them to the <default encoding> + "s": For Unicode objects: auto convert them to the <default encoding> and return a pointer to the object's <defencstr> buffer. - 's#': Access to the Unicode object via the bf_getreadbuf buffer interface + "s#": Access to the Unicode object via the bf_getreadbuf buffer interface (see Buffer Interface); note that the length relates to the buffer length, not the Unicode string length (this may be different depending on the Internal Format). - 't#': Access to the Unicode object via the bf_getcharbuf buffer interface + "t#": Access to the Unicode object via the bf_getcharbuf buffer interface (see Buffer Interface); note that the length relates to the buffer length, not necessarily to the Unicode string length (this may be different depending on the <default encoding>). + "es": + Takes two parameters: encoding (const char *) and + buffer (char **). + + The input object is first coerced to Unicode in the usual way + and then encoded into a string using the given encoding. + + On output, a buffer of the needed size is allocated and + returned through *buffer as NULL-terminated string. + The encoded may not contain embedded NULL characters. + The caller is responsible for free()ing the allocated *buffer + after usage. + + "es#": + Takes three parameters: encoding (const char *), + buffer (char **) and buffer_len (int *). + + The input object is first coerced to Unicode in the usual way + and then encoded into a string using the given encoding. + + If *buffer is non-NULL, *buffer_len must be set to sizeof(buffer) + on input. Output is then copied to *buffer. + + If *buffer is NULL, a buffer of the needed size is + allocated and output copied into it. *buffer is then + updated to point to the allocated memory area. The caller + is responsible for free()ing *buffer after usage. + + In both cases *buffer_len is updated to the number of + characters written (excluding the trailing NULL-byte). + The output buffer is assured to be NULL-terminated. + +Examples: + +Using "es#" with auto-allocation: + + static PyObject * + test_parser(PyObject *self, + PyObject *args) + { + PyObject *str; + const char *encoding = "latin-1"; + char *buffer = NULL; + int buffer_len = 0; + + if (!PyArg_ParseTuple(args, "es#:test_parser", + encoding, &buffer, &buffer_len)) + return NULL; + if (!buffer) { + PyErr_SetString(PyExc_SystemError, + "buffer is NULL"); + return NULL; + } + str = PyString_FromStringAndSize(buffer, buffer_len); + free(buffer); + return str; + } + +Using "es" with auto-allocation returning a NULL-terminated string: + + static PyObject * + test_parser(PyObject *self, + PyObject *args) + { + PyObject *str; + const char *encoding = "latin-1"; + char *buffer = NULL; + + if (!PyArg_ParseTuple(args, "es:test_parser", + encoding, &buffer)) + return NULL; + if (!buffer) { + PyErr_SetString(PyExc_SystemError, + "buffer is NULL"); + return NULL; + } + str = PyString_FromString(buffer); + free(buffer); + return str; + } + +Using "es#" with a pre-allocated buffer: + + static PyObject * + test_parser(PyObject *self, + PyObject *args) + { + PyObject *str; + const char *encoding = "latin-1"; + char _buffer[10]; + char *buffer = _buffer; + int buffer_len = sizeof(_buffer); + + if (!PyArg_ParseTuple(args, "es#:test_parser", + encoding, &buffer, &buffer_len)) + return NULL; + if (!buffer) { + PyErr_SetString(PyExc_SystemError, + "buffer is NULL"); + return NULL; + } + str = PyString_FromStringAndSize(buffer, buffer_len); + return str; + } + File/Stream Output: ------------------- @@ -837,6 +942,7 @@ History of this Proposal: ------------------------- +1.3: Added new "es" and "es#" parser markers 1.2: Removed POD about codecs.open() 1.1: Added note about comparisons and hash values. Added note about case mapping algorithms. Changed stream codecs .read() and Only in CVS-Python/Objects: .#stringobject.c.2.59 Only in CVS-Python/Objects: stringobject.c.orig diff -u -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS CVS-Python/Python/getargs.c Python+Unicode/Python/getargs.c --- CVS-Python/Python/getargs.c Sat Mar 11 10:55:21 2000 +++ Python+Unicode/Python/getargs.c Fri Mar 24 20:22:26 2000 @@ -178,6 +178,8 @@ } else if (level != 0) ; /* Pass */ + else if (c == 'e') + ; /* Pass */ else if (isalpha(c)) max++; else if (c == '|') @@ -654,6 +656,122 @@ break; } + case 'e': /* encoded string */ + { + char **buffer; + const char *encoding; + PyObject *u, *s; + int size; + + /* Get 'e' parameter: the encoding name */ + encoding = (const char *)va_arg(*p_va, const char *); + if (encoding == NULL) + return "(encoding is NULL)"; + + /* Get 's' parameter: the output buffer to use */ + if (*format != 's') + return "(unkown parser marker combination)"; + buffer = (char **)va_arg(*p_va, char **); + format++; + if (buffer == NULL) + return "(buffer is NULL)"; + + /* Convert object to Unicode */ + u = PyUnicode_FromObject(arg); + if (u == NULL) + return "string, unicode or text buffer"; + + /* Encode object; use default error handling */ + s = PyUnicode_AsEncodedString(u, + encoding, + NULL); + Py_DECREF(u); + if (s == NULL) + return "(encoding failed)"; + if (!PyString_Check(s)) { + Py_DECREF(s); + return "(encoder failed to return a string)"; + } + size = PyString_GET_SIZE(s); + + /* Write output; output is guaranteed to be + 0-terminated */ + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer + of the needed size is allocated and + the data copied into it; *buffer is + updated to point to the new buffer; + the caller is responsible for + free()ing it after usage + + - if *buffer is not NULL, the data + is copied to *buffer; *buffer_len + has to be set to the size of the + buffer on input; buffer overflow is + signalled with an error; buffer has + to provide enough room for the + encoded string plus the trailing + 0-byte + + - in both cases, *buffer_len is + updated to the size of the buffer + /excluding/ the trailing 0-byte + + */ + int *buffer_len = va_arg(*p_va, int *); + + format++; + if (buffer_len == NULL) + return "(buffer_len is NULL)"; + if (*buffer == NULL) { + *buffer = PyMem_NEW(char, size + 1); + if (*buffer == NULL) { + Py_DECREF(s); + return "(memory error)"; + } + } else { + if (size + 1 > *buffer_len) { + Py_DECREF(s); + return "(buffer overflow)"; + } + } + memcpy(*buffer, + PyString_AS_STRING(s), + size + 1); + *buffer_len = size; + } else { + /* Using a 0-terminated buffer: + + - the encoded string has to be + 0-terminated for this variant to + work; if it is not, an error raised + + - a new buffer of the needed size + is allocated and the data copied + into it; *buffer is updated to + point to the new buffer; the caller + is responsible for free()ing it + after usage + + */ + if (strlen(PyString_AS_STRING(s)) != size) + return "(encoded string without "\ + "NULL bytes)"; + *buffer = PyMem_NEW(char, size + 1); + if (*buffer == NULL) { + Py_DECREF(s); + return "(memory error)"; + } + memcpy(*buffer, + PyString_AS_STRING(s), + size + 1); + } + Py_DECREF(s); + break; + } + case 'S': /* string object */ { PyObject **p = va_arg(*p_va, PyObject **);