[Python-checkins] cpython: Implement PEP 393.

http://hg.python.org/cpython/rev/8beaa9a37387 changeset: 72475:8beaa9a37387 user: Martin v. Löwis <martin@v.loewis.de> date: Wed Sep 28 07:41:54 2011 +0200 summary: Implement PEP 393. files: Doc/c-api/unicode.rst | 9 + Include/Python.h | 5 + Include/complexobject.h | 5 +- Include/floatobject.h | 5 +- Include/longobject.h | 6 +- Include/pyerrors.h | 6 + Include/pyport.h | 3 + Include/unicodeobject.h | 783 +- Lib/json/decoder.py | 3 +- Lib/test/json_tests/test_scanstring.py | 11 +- Lib/test/test_codeccallbacks.py | 7 +- Lib/test/test_codecs.py | 4 + Lib/test/test_peepholer.py | 4 - Lib/test/test_re.py | 7 + Lib/test/test_sys.py | 38 +- Lib/test/test_unicode.py | 41 +- Makefile.pre.in | 6 +- Misc/NEWS | 2 + Modules/_codecsmodule.c | 8 +- Modules/_csv.c | 2 +- Modules/_ctypes/_ctypes.c | 6 +- Modules/_ctypes/callproc.c | 8 - Modules/_ctypes/cfield.c | 64 +- Modules/_cursesmodule.c | 7 +- Modules/_datetimemodule.c | 13 +- Modules/_dbmmodule.c | 12 +- Modules/_elementtree.c | 31 +- Modules/_io/_iomodule.h | 2 +- Modules/_io/stringio.c | 69 +- Modules/_io/textio.c | 352 +- Modules/_json.c | 252 +- Modules/_pickle.c | 4 +- Modules/_sqlite/connection.c | 19 +- Modules/_sre.c | 382 +- Modules/_testcapimodule.c | 2 +- Modules/_tkinter.c | 70 +- Modules/arraymodule.c | 8 +- Modules/md5module.c | 10 +- Modules/operator.c | 27 +- Modules/pyexpat.c | 11 +- Modules/sha1module.c | 10 +- Modules/sha256module.c | 10 +- Modules/sha512module.c | 10 +- Modules/sre.h | 4 +- Modules/syslogmodule.c | 14 +- Modules/unicodedata.c | 28 +- Modules/zipimport.c | 141 +- Objects/abstract.c | 4 +- Objects/bytearrayobject.c | 147 +- Objects/bytesobject.c | 127 +- Objects/codeobject.c | 15 +- Objects/complexobject.c | 19 +- Objects/dictobject.c | 20 +- Objects/exceptions.c | 26 +- Objects/fileobject.c | 17 +- Objects/floatobject.c | 19 +- Objects/longobject.c | 84 +- Objects/moduleobject.c | 9 +- Objects/object.c | 10 +- Objects/setobject.c | 40 +- Objects/stringlib/count.h | 9 +- Objects/stringlib/eq.h | 23 +- Objects/stringlib/fastsearch.h | 4 +- Objects/stringlib/find.h | 31 +- Objects/stringlib/formatter.h | 1516 -- Objects/stringlib/localeutil.h | 27 +- Objects/stringlib/partition.h | 12 +- Objects/stringlib/split.h | 26 +- Objects/stringlib/string_format.h | 1385 -- Objects/stringlib/stringdefs.h | 2 + Objects/stringlib/ucs1lib.h | 35 + Objects/stringlib/ucs2lib.h | 34 + Objects/stringlib/ucs4lib.h | 34 + Objects/stringlib/undef.h | 10 + Objects/stringlib/unicode_format.h | 1416 ++ Objects/stringlib/unicodedefs.h | 2 + Objects/typeobject.c | 18 +- Objects/unicodeobject.c | 6112 ++++++++--- Objects/uniops.h | 91 + PC/_subprocess.c | 61 +- PC/import_nt.c | 2 +- PC/msvcrtmodule.c | 8 +- PC/pyconfig.h | 4 - PC/winreg.c | 8 +- Parser/tokenizer.c | 6 +- Python/_warnings.c | 16 +- Python/ast.c | 61 +- Python/bltinmodule.c | 26 +- Python/ceval.c | 17 +- Python/codecs.c | 44 +- Python/compile.c | 89 +- Python/errors.c | 4 +- Python/formatter_unicode.c | 1445 ++- Python/getargs.c | 46 +- Python/import.c | 347 +- Python/marshal.c | 4 +- Python/peephole.c | 18 - Python/symtable.c | 8 +- Python/traceback.c | 59 +- Tools/gdb/libpython.py | 27 +- configure | 65 +- configure.in | 46 +- pyconfig.h.in | 6 - 103 files changed, 9492 insertions(+), 6770 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1072,6 +1072,15 @@ occurred and an exception has been set. +.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, Py_ssize_t start, Py_ssize_t end, int direction) + + Return the first position of the character *ch* in ``str[start:end]`` using + the given *direction* (*direction* == 1 means to do a forward search, + *direction* == -1 a backward search). The return value is the index of the + first match; a value of ``-1`` indicates that no match was found, and ``-2`` + indicates that an error occurred and an exception has been set. + + .. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) Return the number of non-overlapping occurrences of *substr* in diff --git a/Include/Python.h b/Include/Python.h --- a/Include/Python.h +++ b/Include/Python.h @@ -160,4 +160,9 @@ #define PyDoc_STR(str) "" #endif +#define PY_ARRAY_LENGTH(array) (sizeof(array) / sizeof((array)[0])) + +#define PY_MIN(x, y) (((x) > (y)) ? (y) : (x)) +#define PY_MAX(x, y) (((x) > (y)) ? (x) : (y)) + #endif /* !Py_PYTHON_H */ diff --git a/Include/complexobject.h b/Include/complexobject.h --- a/Include/complexobject.h +++ b/Include/complexobject.h @@ -64,8 +64,9 @@ (Advanced String Formatting). */ #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif #ifdef __cplusplus diff --git a/Include/floatobject.h b/Include/floatobject.h --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -113,8 +113,9 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Include/longobject.h b/Include/longobject.h --- a/Include/longobject.h +++ b/Include/longobject.h @@ -80,6 +80,7 @@ PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int); +PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); #endif #ifndef Py_LIMITED_API @@ -155,8 +156,9 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ /* These aren't really part of the long object, but they're handy. The diff --git a/Include/pyerrors.h b/Include/pyerrors.h --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -301,6 +301,12 @@ Py_ssize_t end, const char *reason /* UTF-8 encoded string */ ); +PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, + Py_ssize_t end, + const char *reason /* UTF-8 encoded string */ + ); #endif /* get the encoding attribute */ diff --git a/Include/pyport.h b/Include/pyport.h --- a/Include/pyport.h +++ b/Include/pyport.h @@ -286,12 +286,15 @@ /* fastest possible local call under MSVC */ #define Py_LOCAL(type) static type __fastcall #define Py_LOCAL_INLINE(type) static __inline type __fastcall +#define Py_LOCAL_CALLBACK(name) (__fastcall *name) #elif defined(USE_INLINE) #define Py_LOCAL(type) static type #define Py_LOCAL_INLINE(type) static inline type +#define Py_LOCAL_CALLBACK(name) (*name) #else #define Py_LOCAL(type) static type #define Py_LOCAL_INLINE(type) static type +#define Py_LOCAL_CALLBACK(name) (*name) #endif /* Py_MEMCPY can be used instead of memcpy in cases where the copied blocks diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,16 +64,15 @@ /* Python 3.x requires unicode */ #define Py_USING_UNICODE -/* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is - properly set, but the default rules below doesn't set it. I'll - sort this out some other day -- fredrik@pythonware.com */ - -#ifndef Py_UNICODE_SIZE -#error Must define Py_UNICODE_SIZE +#ifndef SIZEOF_WCHAR_T +#error Must define SIZEOF_WCHAR_T #endif -/* Setting Py_UNICODE_WIDE enables UCS-4 storage. Otherwise, Unicode - strings are stored as UCS-2 (with limited support for UTF-16) */ +#define Py_UNICODE_SIZE SIZEOF_WCHAR_T + +/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. + Otherwise, Unicode strings are stored as UCS-2 (with limited support + for UTF-16) */ #if Py_UNICODE_SIZE >= 4 #define Py_UNICODE_WIDE @@ -84,19 +83,14 @@ /* #define HAVE_WCHAR_H */ /* #define HAVE_USABLE_WCHAR_T */ -/* Defaults for various platforms */ -#ifndef PY_UNICODE_TYPE +/* Py_UNICODE was the native Unicode storage format (code unit) used by + Python and represents a single Unicode element in the Unicode type. + With PEP 393, Py_UNICODE is deprected and replaced with a + typedef to wchar_t. */ -/* Windows has a usable wchar_t type (unless we're using UCS-4) */ -# if defined(MS_WIN32) && Py_UNICODE_SIZE == 2 -# define HAVE_USABLE_WCHAR_T -# define PY_UNICODE_TYPE wchar_t -# endif - -# if defined(Py_UNICODE_WIDE) -# define PY_UNICODE_TYPE Py_UCS4 -# endif - +#ifndef Py_LIMITED_API +#define PY_UNICODE_TYPE wchar_t +typedef wchar_t Py_UNICODE; #endif /* If the compiler provides a wchar_t type we try to support it @@ -109,7 +103,7 @@ # endif #endif -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#if defined(MS_WINDOWS) # define HAVE_MBCS #endif @@ -121,200 +115,18 @@ # include <wchar.h> #endif -/* - * Use this typedef when you need to represent a UTF-16 surrogate pair - * as single unsigned integer. - */ +/* Py_UCS4 and Py_UCS2 are typdefs for the respecitve + unicode representations. */ #if SIZEOF_INT >= 4 typedef unsigned int Py_UCS4; #elif SIZEOF_LONG >= 4 typedef unsigned long Py_UCS4; +#else +#error "Could not find a proper typedef for Py_UCS4" #endif -/* Py_UNICODE is the native Unicode storage format (code unit) used by - Python and represents a single Unicode element in the Unicode - type. */ - -#ifndef Py_LIMITED_API -typedef PY_UNICODE_TYPE Py_UNICODE; -#endif - -/* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */ - -/* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds - produce different external names and thus cause import errors in - case Python interpreters and extensions with mixed compiled in - Unicode width assumptions are combined. */ - -#ifndef Py_UNICODE_WIDE - -# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString -# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString -# define PyUnicode_AsDecodedObject PyUnicodeUCS2_AsDecodedObject -# define PyUnicode_AsDecodedUnicode PyUnicodeUCS2_AsDecodedUnicode -# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject -# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString -# define PyUnicode_AsEncodedUnicode PyUnicodeUCS2_AsEncodedUnicode -# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String -# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString -# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String -# define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String -# define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String -# define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode -# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString -# define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar -# define PyUnicode_AsWideCharString PyUnicodeUCS2_AsWideCharString -# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist -# define PyUnicode_Compare PyUnicodeUCS2_Compare -# define PyUnicode_CompareWithASCIIString PyUnicodeUCS2_CompareWithASCIIString -# define PyUnicode_Concat PyUnicodeUCS2_Concat -# define PyUnicode_Append PyUnicodeUCS2_Append -# define PyUnicode_AppendAndDel PyUnicodeUCS2_AppendAndDel -# define PyUnicode_Contains PyUnicodeUCS2_Contains -# define PyUnicode_Count PyUnicodeUCS2_Count -# define PyUnicode_Decode PyUnicodeUCS2_Decode -# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII -# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap -# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1 -# define PyUnicode_DecodeFSDefault PyUnicodeUCS2_DecodeFSDefault -# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS2_DecodeFSDefaultAndSize -# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape -# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32 -# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful -# define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16 -# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful -# define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8 -# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful -# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape -# define PyUnicode_Encode PyUnicodeUCS2_Encode -# define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII -# define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap -# define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal -# define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1 -# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape -# define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32 -# define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16 -# define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8 -# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape -# define PyUnicode_Find PyUnicodeUCS2_Find -# define PyUnicode_Format PyUnicodeUCS2_Format -# define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject -# define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat -# define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV -# define PyUnicode_FromObject PyUnicodeUCS2_FromObject -# define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal -# define PyUnicode_FromString PyUnicodeUCS2_FromString -# define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize -# define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode -# define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar -# define PyUnicode_FSConverter PyUnicodeUCS2_FSConverter -# define PyUnicode_FSDecoder PyUnicodeUCS2_FSDecoder -# define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding -# define PyUnicode_GetMax PyUnicodeUCS2_GetMax -# define PyUnicode_GetSize PyUnicodeUCS2_GetSize -# define PyUnicode_IsIdentifier PyUnicodeUCS2_IsIdentifier -# define PyUnicode_Join PyUnicodeUCS2_Join -# define PyUnicode_Partition PyUnicodeUCS2_Partition -# define PyUnicode_RPartition PyUnicodeUCS2_RPartition -# define PyUnicode_RSplit PyUnicodeUCS2_RSplit -# define PyUnicode_Replace PyUnicodeUCS2_Replace -# define PyUnicode_Resize PyUnicodeUCS2_Resize -# define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare -# define PyUnicode_Split PyUnicodeUCS2_Split -# define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines -# define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch -# define PyUnicode_Translate PyUnicodeUCS2_Translate -# define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap -# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString -# define _PyUnicode_Fini _PyUnicodeUCS2_Fini -# define _PyUnicode_Init _PyUnicodeUCS2_Init -# define PyUnicode_strdup PyUnicodeUCS2_strdup - -#else - -# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString -# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString -# define PyUnicode_AsDecodedObject PyUnicodeUCS4_AsDecodedObject -# define PyUnicode_AsDecodedUnicode PyUnicodeUCS4_AsDecodedUnicode -# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject -# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString -# define PyUnicode_AsEncodedUnicode PyUnicodeUCS4_AsEncodedUnicode -# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String -# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString -# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String -# define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String -# define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String -# define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode -# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString -# define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar -# define PyUnicode_AsWideCharString PyUnicodeUCS4_AsWideCharString -# define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist -# define PyUnicode_Compare PyUnicodeUCS4_Compare -# define PyUnicode_CompareWithASCIIString PyUnicodeUCS4_CompareWithASCIIString -# define PyUnicode_Concat PyUnicodeUCS4_Concat -# define PyUnicode_Append PyUnicodeUCS4_Append -# define PyUnicode_AppendAndDel PyUnicodeUCS4_AppendAndDel -# define PyUnicode_Contains PyUnicodeUCS4_Contains -# define PyUnicode_Count PyUnicodeUCS4_Count -# define PyUnicode_Decode PyUnicodeUCS4_Decode -# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII -# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap -# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1 -# define PyUnicode_DecodeFSDefault PyUnicodeUCS4_DecodeFSDefault -# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS4_DecodeFSDefaultAndSize -# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape -# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32 -# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful -# define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16 -# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful -# define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8 -# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful -# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape -# define PyUnicode_Encode PyUnicodeUCS4_Encode -# define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII -# define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap -# define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal -# define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1 -# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape -# define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32 -# define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16 -# define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8 -# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape -# define PyUnicode_Find PyUnicodeUCS4_Find -# define PyUnicode_Format PyUnicodeUCS4_Format -# define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject -# define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat -# define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV -# define PyUnicode_FromObject PyUnicodeUCS4_FromObject -# define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal -# define PyUnicode_FromString PyUnicodeUCS4_FromString -# define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize -# define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode -# define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar -# define PyUnicode_FSConverter PyUnicodeUCS4_FSConverter -# define PyUnicode_FSDecoder PyUnicodeUCS4_FSDecoder -# define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding -# define PyUnicode_GetMax PyUnicodeUCS4_GetMax -# define PyUnicode_GetSize PyUnicodeUCS4_GetSize -# define PyUnicode_IsIdentifier PyUnicodeUCS4_IsIdentifier -# define PyUnicode_Join PyUnicodeUCS4_Join -# define PyUnicode_Partition PyUnicodeUCS4_Partition -# define PyUnicode_RPartition PyUnicodeUCS4_RPartition -# define PyUnicode_RSplit PyUnicodeUCS4_RSplit -# define PyUnicode_Replace PyUnicodeUCS4_Replace -# define PyUnicode_Resize PyUnicodeUCS4_Resize -# define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare -# define PyUnicode_Split PyUnicodeUCS4_Split -# define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines -# define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch -# define PyUnicode_Translate PyUnicodeUCS4_Translate -# define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap -# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString -# define _PyUnicode_Fini _PyUnicodeUCS4_Fini -# define _PyUnicode_Init _PyUnicodeUCS4_Init -# define PyUnicode_strdup PyUnicodeUCS4_strdup - -#endif +typedef unsigned short Py_UCS2; +typedef unsigned char Py_UCS1; /* --- Internal Unicode Operations ---------------------------------------- */ @@ -354,7 +166,7 @@ Py_UNICODE_ISDIGIT(ch) || \ Py_UNICODE_ISNUMERIC(ch)) -#define Py_UNICODE_COPY(target, source, length) \ +#define Py_UNICODE_COPY(target, source, length) \ Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE)) #define Py_UNICODE_FILL(target, value, length) \ @@ -375,9 +187,10 @@ valid, and the substring must not be empty. */ #define Py_UNICODE_MATCH(string, offset, substring) \ - ((*((string)->str + (offset)) == *((substring)->str)) && \ - ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \ - !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE))) + ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \ + ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \ + !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE))) + #endif /* Py_LIMITED_API */ #ifdef __cplusplus @@ -387,41 +200,303 @@ /* --- Unicode Type ------------------------------------------------------- */ #ifndef Py_LIMITED_API + +/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject + structure. state.ascii and state.compact are set, and the data + immediately follow the structure. utf8_length and wstr_length can be found + in the length field; the utf8 pointer is equal to the data pointer. */ typedef struct { PyObject_HEAD - Py_ssize_t length; /* Length of raw Unicode data in buffer */ - Py_UNICODE *str; /* Raw Unicode buffer */ + Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ - int state; /* != 0 if interned. In this case the two - * references from the dictionary to this object - * are *not* counted in ob_refcnt. */ - PyObject *defenc; /* (Default) Encoded version as Python - string, or NULL; this is used for - implementing the buffer protocol */ + struct { + /* + SSTATE_NOT_INTERNED (0) + SSTATE_INTERNED_MORTAL (1) + SSTATE_INTERNED_IMMORTAL (2) + + If interned != SSTATE_NOT_INTERNED, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + */ + unsigned int interned:2; + /* Character size: + + PyUnicode_WCHAR_KIND (0): wchar_t* + PyUnicode_1BYTE_KIND (1): Py_UCS1* + PyUnicode_2BYTE_KIND (2): Py_UCS2* + PyUnicode_4BYTE_KIND (3): Py_UCS4* + */ + unsigned int kind:2; + /* Compact is with respect to the allocation scheme. Compact unicode + objects only require one memory block while non-compact objects use + one block for the PyUnicodeObject struct and another for its data + buffer. */ + unsigned int compact:1; + /* Compact objects which are ASCII-only also have the state.compact + flag set, and use the PyASCIIObject struct. */ + unsigned int ascii:1; + /* The ready flag indicates whether the object layout is initialized + completely. This means that this is either a compact object, or + the data pointer is filled out. The bit is redundant, and helps + to minimize the test in PyUnicode_IS_READY(). */ + unsigned int ready:1; + } state; + wchar_t *wstr; /* wchar_t representation (null-terminated) */ +} PyASCIIObject; + +/* Non-ASCII strings allocated through PyUnicode_New use the + PyCompactUnicodeOject structure. state.compact is set, and the data + immediately follow the structure. */ +typedef struct { + PyASCIIObject _base; + Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the + * terminating \0. */ + char *utf8; /* UTF-8 representation (null-terminated) */ + Py_ssize_t wstr_length; /* Number of code points in wstr, possible + * surrogates count as two code points. */ +} PyCompactUnicodeObject; + +/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the + PyUnicodeObject structure. The actual string data is initially in the wstr + block, and copied into the data block using PyUnicode_Ready. */ +typedef struct { + PyCompactUnicodeObject _base; + union { + void *any; + Py_UCS1 *latin1; + Py_UCS2 *ucs2; + Py_UCS4 *ucs4; + } data; /* Canonical, smallest-form Unicode buffer */ } PyUnicodeObject; #endif PyAPI_DATA(PyTypeObject) PyUnicode_Type; PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type; -#define SSTATE_NOT_INTERNED 0 -#define SSTATE_INTERNED_MORTAL 1 -#define SSTATE_INTERNED_IMMORTAL 2 - #define PyUnicode_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type) /* Fast access macros */ #ifndef Py_LIMITED_API + +#define PyUnicode_WSTR_LENGTH(op) \ + (((PyASCIIObject*)op)->state.ascii ? \ + ((PyASCIIObject*)op)->length : \ + ((PyCompactUnicodeObject*)op)->wstr_length) + +/* Returns the deprecated Py_UNICODE representation's size in code units + (this includes surrogate pairs as 2 units). + If the Py_UNICODE representation is not available, it will be computed + on request. Use PyUnicode_GET_LENGTH() for the length in code points. */ + #define PyUnicode_GET_SIZE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length)) + (assert(PyUnicode_Check(op)), \ + (((PyASCIIObject *)(op))->wstr) ? \ + PyUnicode_WSTR_LENGTH(op) : \ + ((void)PyUnicode_AsUnicode((PyObject *)(op)), \ + PyUnicode_WSTR_LENGTH(op))) + #define PyUnicode_GET_DATA_SIZE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))) + (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE) + +/* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE + representation on demand. Using this macro is very inefficient now, + try to port your code to use the new PyUnicode_*BYTE_DATA() macros or + use PyUnicode_WRITE() and PyUnicode_READ(). */ + #define PyUnicode_AS_UNICODE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->str)) + (assert(PyUnicode_Check(op)), \ + (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \ + PyUnicode_AsUnicode((PyObject *)(op))) + #define PyUnicode_AS_DATA(op) \ - (assert(PyUnicode_Check(op)),((const char *)((PyUnicodeObject *)(op))->str)) + ((const char *)(PyUnicode_AS_UNICODE(op))) + + +/* --- Flexible String Representaion Helper Macros (PEP 393) -------------- */ + +/* Values for PyUnicodeObject.state: */ + +/* Interning state. */ +#define SSTATE_NOT_INTERNED 0 +#define SSTATE_INTERNED_MORTAL 1 +#define SSTATE_INTERNED_IMMORTAL 2 + +#define PyUnicode_IS_COMPACT_ASCII(op) (((PyASCIIObject*)op)->state.ascii) + +/* String contains only wstr byte characters. This is only possible + when the string was created with a legacy API and PyUnicode_Ready() + has not been called yet. */ +#define PyUnicode_WCHAR_KIND 0 + +/* Return values of the PyUnicode_KIND() macro: */ + +#define PyUnicode_1BYTE_KIND 1 +#define PyUnicode_2BYTE_KIND 2 +#define PyUnicode_4BYTE_KIND 3 + + +/* Return the number of bytes the string uses to represent single characters, + this can be 1, 2 or 4. */ +#define PyUnicode_CHARACTER_SIZE(op) \ + (1 << (PyUnicode_KIND(op) - 1)) + +/* Return pointers to the canonical representation casted as unsigned char, + Py_UCS2, or Py_UCS4 for direct character access. + No checks are performed, use PyUnicode_CHARACTER_SIZE or + PyUnicode_KIND() before to ensure these will work correctly. */ + +#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op)) +#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op)) +#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op)) + +/* Return true if the string is compact or 0 if not. + No type checks or Ready calls are performed. */ +#define PyUnicode_IS_COMPACT(op) \ + (((PyASCIIObject*)(op))->state.compact) + +/* Return one of the PyUnicode_*_KIND values defined above. */ +#define PyUnicode_KIND(op) \ + (assert(PyUnicode_Check(op)), \ + assert(PyUnicode_IS_READY(op)), \ + ((PyASCIIObject *)(op))->state.kind) + +/* Return a void pointer to the raw unicode buffer. */ +#define _PyUnicode_COMPACT_DATA(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((void*)((PyASCIIObject*)(op) + 1)) : \ + ((void*)((PyCompactUnicodeObject*)(op) + 1))) + +#define _PyUnicode_NONCOMPACT_DATA(op) \ + (assert(((PyUnicodeObject*)(op))->data.any), \ + ((((PyUnicodeObject *)(op))->data.any))) + +#define PyUnicode_DATA(op) \ + (assert(PyUnicode_Check(op)), \ + PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \ + _PyUnicode_NONCOMPACT_DATA(op)) + +#define _PyUnicode_UTF8(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((char*)((PyASCIIObject*)(op) + 1)) : \ + ((PyCompactUnicodeObject*)(op))->utf8) + +#define _PyUnicode_UTF8_LENGTH(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((PyASCIIObject*)(op))->length : \ + ((PyCompactUnicodeObject*)(op))->utf8_length) + +/* Compute (index * char_size) where char_size is 2 ** (kind - 1). + + The index is a character index, the result is a size in bytes. */ +#define PyUnicode_KIND_SIZE(kind, index) ((index) << ((kind) - 1)) + +/* In the access macros below, "kind" may be evaluated more than once. + All other macro parameters are evaluated exactly once, so it is safe + to put side effects into them (such as increasing the index). */ + +/* Write into the canonical representation, this macro does not do any sanity + checks and is intended for usage in loops. The caller should cache the + kind and data pointers optained form other macro calls. + index is the index in the string (starts at 0) and value is the new + code point value which shoule be written to that location. */ +#define PyUnicode_WRITE(kind, data, index, value) \ + do { \ + switch ((kind)) { \ + case PyUnicode_1BYTE_KIND: { \ + ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \ + break; \ + } \ + case PyUnicode_2BYTE_KIND: { \ + ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \ + break; \ + } \ + default: { \ + assert((kind) == PyUnicode_4BYTE_KIND); \ + ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \ + } \ + } \ + } while (0) + +/* Read a code point form the string's canonical representation. No checks + or ready calls are performed. */ +#define PyUnicode_READ(kind, data, index) \ + ((Py_UCS4) \ + ((kind) == PyUnicode_1BYTE_KIND ? \ + ((const unsigned char *)(data))[(index)] : \ + ((kind) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(data))[(index)] : \ + ((const Py_UCS4 *)(data))[(index)] \ + ) \ + )) + +/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it + calls PyUnicode_KIND() and might call it twice. For single reads, use + PyUnicode_READ_CHAR, for multiple consecutive reads callers should + cache kind and use PyUnicode_READ instead. */ +#define PyUnicode_READ_CHAR(unicode, index) \ + ((Py_UCS4) \ + (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \ + ((const unsigned char *)(PyUnicode_DATA((unicode))))[(index)] : \ + (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \ + ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \ + ) \ + )) + +/* Returns the length of the unicode string. The caller has to make sure that + the string has it's canonical representation set before calling + this macro. Call PyUnicode_(FAST_)Ready to ensure that. */ +#define PyUnicode_GET_LENGTH(op) \ + (assert(PyUnicode_Check(op)), \ + assert(PyUnicode_IS_READY(op)), \ + ((PyASCIIObject *)(op))->length) + + +/* Fast check to determine whether an object is ready. Equivalent to + PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */ + +#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready) + +/* PyUnicode_READY() does less work than PyUnicode_Ready() in the best + case. If the canonical representation is not yet set, it will still call + PyUnicode_Ready(). + Returns 0 on success and -1 on errors. */ +#define PyUnicode_READY(op) \ + (assert(PyUnicode_Check(op)), \ + (PyUnicode_IS_READY(op) ? \ + 0 : _PyUnicode_Ready((PyUnicodeObject *)(op)))) + +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + const from_type *iter_; to_type *to_; \ + for (iter_ = (begin), to_ = (to_type *)(to); \ + iter_ < (end); \ + ++iter_, ++to_) { \ + *to_ = (to_type)*iter_; \ + } \ + } while (0) + +/* Return a maximum character value which is suitable for creating another + string based on op. This is always an approximation but more efficient + than interating over the string. */ +#define PyUnicode_MAX_CHAR_VALUE(op) \ + (assert(PyUnicode_IS_READY(op)), \ + (PyUnicode_IS_COMPACT_ASCII(op) ? 0x7f: \ + (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \ + (PyUnicode_DATA(op) == (((PyCompactUnicodeObject *)(op))->utf8) ? \ + (0x7fU) : (0xffU) \ + ) : \ + (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \ + (0xffffU) : (0x10ffffU) \ + )))) + #endif /* --- Constants ---------------------------------------------------------- */ @@ -437,6 +512,52 @@ /* --- Plain Py_UNICODE --------------------------------------------------- */ +/* With PEP 393, this is the recommended way to allocate a new unicode object. + This function will allocate the object and its buffer in a single memory + block. Objects created using this function are not resizable. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) PyUnicode_New( + Py_ssize_t size, /* Number of code points in the new string */ + Py_UCS4 maxchar /* maximum code point value in the string */ + ); +#endif + +/* Initializes the canonical string representation from a the deprected + wstr/Py_UNICODE representation. This function is used to convert + unicode objects which were created using the old API to the new flexible + format introduced with PEP 393. The PyUnicode_READY() macro can be + more efficient if the string is already ready. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PyUnicode_Ready( + PyUnicodeObject *unicode /* Unicode object */ + ); +#endif + +/* Copy character from one unicode object into another, this function performs + character conversion when nessesary and falls back to memcpy if possible. + Return -1 and raise an exception on error, return 0 on success. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyUnicode_CopyCharacters( + PyObject *to, + Py_ssize_t to_start, + PyObject *from, + Py_ssize_t from_start, + Py_ssize_t how_many + ); +#endif + +/* Find the maximum code point and count the number of surrogate pairs so a + correct string length can be computed before converting a string to UCS4. + This function counts single surrogates as a character and not as a pair. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PyUnicode_FindMaxCharAndNumSurrogatePairs( + const wchar_t *begin, + const wchar_t *end, + Py_UCS4 *maxchar, + Py_ssize_t *num_surrogates + ); +#endif + /* Create a Unicode Object from the Py_UNICODE buffer u of the given size. @@ -461,13 +582,43 @@ ); /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated - UTF-8 encoded bytes */ + UTF-8 encoded bytes. The size is determined with strlen(). */ PyAPI_FUNC(PyObject*) PyUnicode_FromString( const char *u /* UTF-8 encoded string */ ); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( + int kind, + const void *buffer, + Py_ssize_t size); +#endif + +PyAPI_FUNC(PyObject*) PyUnicode_Substring( + PyObject *str, + Py_ssize_t start, + Py_ssize_t end); + +/* Copy the string into a UCS4 buffer including the null character is copy_null + is set. Return NULL and raise an exception on error. Raise a ValueError if + the buffer is smaller than the string. Return buffer on success. + + buflen is the length of the buffer in (Py_UCS4) characters. */ +PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4( + PyObject *unicode, + Py_UCS4* buffer, + Py_ssize_t buflen, + int copy_null); + +/* Copy the string into a UCS4 buffer. A new buffer is allocated using + * PyMem_Malloc; if this fails, NULL is returned with a memory error + exception set. */ +PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode); + /* Return a read-only pointer to the Unicode object's internal - Py_UNICODE buffer. */ + Py_UNICODE buffer. + If the wchar_t/Py_UNICODE representation is not yet available, this + function will calculate it. */ #ifndef Py_LIMITED_API PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( @@ -475,12 +626,47 @@ ); #endif +/* Return a read-only pointer to the Unicode object's internal + Py_UNICODE buffer and save the length at size. + If the wchar_t/Py_UNICODE representation is not yet available, this + function will calculate it. */ + +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( + PyObject *unicode, /* Unicode object */ + Py_ssize_t *size /* location where to save the length */ + ); +#endif + /* Get the length of the Unicode object. */ +PyAPI_FUNC(Py_ssize_t) PyUnicode_GetLength( + PyObject *unicode +); + +/* Get the number of Py_UNICODE units in the + string representation. */ + PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize( PyObject *unicode /* Unicode object */ ); +/* Read a character from the string. */ + +PyAPI_FUNC(Py_UCS4) PyUnicode_ReadChar( + PyObject *unicode, + Py_ssize_t index + ); + +/* Write a character to the string. The string must have been created through + PyUnicode_New, must not be shared, and must not have been hashed yet. */ + +PyAPI_FUNC(int) PyUnicode_WriteChar( + PyObject *unicode, + Py_ssize_t index, + Py_UCS4 character + ); + #ifndef Py_LIMITED_API /* Get the maximum ordinal for a Unicode character. */ PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void); @@ -558,8 +744,9 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); @@ -572,7 +759,8 @@ #endif /* Use only if you know it's a string */ -#define PyUnicode_CHECK_INTERNED(op) (((PyUnicodeObject *)(op))->state) +#define PyUnicode_CHECK_INTERNED(op) \ + (((PyASCIIObject *)(op))->state.interned) /* --- wchar_t support for platforms which support it --------------------- */ @@ -619,6 +807,8 @@ Py_ssize_t *size /* number of characters of the result */ ); +PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); + #endif /* --- Unicode ordinals --------------------------------------------------- */ @@ -664,49 +854,42 @@ /* --- Manage the default encoding ---------------------------------------- */ -/* Return a Python string holding the default encoded value of the - Unicode object. - - Same as PyUnicode_AsUTF8String() except - the resulting string is cached in the Unicode object for subsequent - usage by this function. The cached version is needed to implement - the character buffer interface and will live (at least) as long as - the Unicode object itself. - - The refcount of the string is *not* incremented. - - *** Exported for internal use by the interpreter only !!! *** - -*/ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString( - PyObject *unicode); -#endif - /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode and the size of the encoded representation in bytes stored in *size. In case of an error, no *size is set. + This funcation caches the UTF-8 encoded string in the unicodeobject + and subsequent calls will return the same string. The memory is relased + when the unicodeobject is deallocated. + + _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to + support the previous internal function with the same behaviour. + *** This API is for interpreter INTERNAL USE ONLY and will likely *** be removed or changed in the future. *** If you need to access the Unicode object as UTF-8 bytes string, *** please use PyUnicode_AsUTF8String() instead. - */ #ifndef Py_LIMITED_API -PyAPI_FUNC(char *) _PyUnicode_AsStringAndSize( +PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize( PyObject *unicode, Py_ssize_t *size); +#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize #endif /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode. + Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation + in the unicodeobject. + + _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to + support the previous internal function with the same behaviour. + Use of this API is DEPRECATED since no size information can be extracted from the returned data. @@ -719,7 +902,8 @@ */ #ifndef Py_LIMITED_API -PyAPI_FUNC(char *) _PyUnicode_AsString(PyObject *unicode); +PyAPI_FUNC(char *) PyUnicode_AsUTF8(PyObject *unicode); +#define _PyUnicode_AsString PyUnicode_AsUTF8 #endif /* Returns "utf-8". */ @@ -846,6 +1030,10 @@ ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( + PyObject *unicode, + const char *errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* number of Py_UNICODE chars to encode */ @@ -1076,6 +1264,10 @@ ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( + PyObject* unicode, + const char* errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ @@ -1100,6 +1292,10 @@ ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString( + PyObject* unicode, + const char* errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ @@ -1252,6 +1448,17 @@ ); #endif +/* Similar to PyUnicode_TransformDecimalToASCII(), but takes a PyUnicodeObject + as argument instead of a raw buffer and length. This function additionally + transforms spaces to ASCII because this is what the callers in longobject, + floatobject, and complexobject did anyways. */ + +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( + PyObject *unicode /* Unicode object */ + ); +#endif + /* --- File system encoding ---------------------------------------------- */ /* ParseTuple converter: encode str objects to bytes using @@ -1439,6 +1646,15 @@ int direction /* Find direction: +1 forward, -1 backward */ ); +/* Like PyUnicode_Find, but search for single character only. */ +PyAPI_FUNC(Py_ssize_t) PyUnicode_FindChar( + PyObject *str, + Py_UCS4 ch, + Py_ssize_t start, + Py_ssize_t end, + int direction + ); + /* Count the number of occurrences of substr in str[start:end]. */ PyAPI_FUNC(Py_ssize_t) PyUnicode_Count( @@ -1542,13 +1758,15 @@ into the string pointed to by buffer. For the argument descriptions, see Objects/stringlib/localeutil.h */ #ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(Py_UNICODE *buffer, - Py_ssize_t n_buffer, - Py_UNICODE *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - const char *thousands_sep); +PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( + int kind, + void *buffer, + Py_ssize_t n_buffer, + void *digits, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + const char *thousands_sep); #endif /* === Characters Type APIs =============================================== */ @@ -1673,6 +1891,43 @@ Py_UNICODE c ); +PyAPI_FUNC(size_t) Py_UCS4_strlen( + const Py_UCS4 *u + ); + +PyAPI_FUNC(Py_UCS4*) Py_UCS4_strcpy( + Py_UCS4 *s1, + const Py_UCS4 *s2); + +PyAPI_FUNC(Py_UCS4*) Py_UCS4_strcat( + Py_UCS4 *s1, const Py_UCS4 *s2); + +PyAPI_FUNC(Py_UCS4*) Py_UCS4_strncpy( + Py_UCS4 *s1, + const Py_UCS4 *s2, + size_t n); + +PyAPI_FUNC(int) Py_UCS4_strcmp( + const Py_UCS4 *s1, + const Py_UCS4 *s2 + ); + +PyAPI_FUNC(int) Py_UCS4_strncmp( + const Py_UCS4 *s1, + const Py_UCS4 *s2, + size_t n + ); + +PyAPI_FUNC(Py_UCS4*) Py_UCS4_strchr( + const Py_UCS4 *s, + Py_UCS4 c + ); + +PyAPI_FUNC(Py_UCS4*) Py_UCS4_strrchr( + const Py_UCS4 *s, + Py_UCS4 c + ); + /* Create a copy of a unicode string ending with a nul character. Return NULL and raise a MemoryError exception on memory allocation failure, otherwise return a new allocated buffer (use PyMem_Free() to free the buffer). */ diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -121,8 +121,7 @@ msg = "Invalid \\uXXXX escape" raise ValueError(errmsg(msg, s, end)) uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + if 0xd800 <= uni <= 0xdbff: msg = "Invalid \\uXXXX\\uXXXX surrogate pair" if not s[end + 5:end + 7] == '\\u': raise ValueError(errmsg(msg, s, end)) diff --git a/Lib/test/json_tests/test_scanstring.py b/Lib/test/json_tests/test_scanstring.py --- a/Lib/test/json_tests/test_scanstring.py +++ b/Lib/test/json_tests/test_scanstring.py @@ -9,14 +9,9 @@ scanstring('"z\\ud834\\udd20x"', 1, True), ('z\U0001d120x', 16)) - if sys.maxunicode == 65535: - self.assertEqual( - scanstring('"z\U0001d120x"', 1, True), - ('z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring('"z\U0001d120x"', 1, True), - ('z\U0001d120x', 5)) + self.assertEqual( + scanstring('"z\U0001d120x"', 1, True), + ('z\U0001d120x', 5)) self.assertEqual( scanstring('"\\u007b"', 1, True), diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,5 +1,6 @@ import test.support, unittest import sys, codecs, html.entities, unicodedata +import ctypes class PosReturn: # this can be used for configurable callbacks @@ -577,8 +578,10 @@ UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")), ("\\uffff", 1) ) - # 1 on UCS-4 builds, 2 on UCS-2 - len_wide = len("\U00010000") + if ctypes.sizeof(ctypes.c_wchar) == 2: + len_wide = 2 + else: + len_wide = 1 self.assertEqual( codecs.backslashreplace_errors( UnicodeEncodeError("ascii", "\U00010000", diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -622,6 +622,10 @@ b"abc\xed\xa0\x80def") self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), "abc\ud800def") + self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + "\U00010fff\uD800") self.assertTrue(codecs.lookup_error("surrogatepass")) class UTF7Test(ReadTest): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -218,10 +218,6 @@ # out of range asm = dis_single('"fuu"[10]') self.assertIn('BINARY_SUBSCR', asm) - # non-BMP char (see #5057) - asm = dis_single('"\U00012345"[0]') - self.assertIn('BINARY_SUBSCR', asm) - def test_folding_of_unaryops_on_constants(self): for line, elem in ( diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -780,6 +780,13 @@ self.assertRaises(OverflowError, _sre.compile, "abc", 0, [long_overflow]) self.assertRaises(TypeError, _sre.compile, {}, 0, []) + def test_search_dot_unicode(self): + self.assertIsNotNone(re.search("123.*-", '123abc-')) + self.assertIsNotNone(re.search("123.*-", '123\xe9-')) + self.assertIsNotNone(re.search("123.*-", '123\u20ac-')) + self.assertIsNotNone(re.search("123.*-", '123\U0010ffff-')) + self.assertIsNotNone(re.search("123.*-", '123\xe9\u20ac\U0010ffff-')) + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR if verbose: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -833,13 +833,39 @@ class newstyleclass(object): pass check(newstyleclass, s) # unicode - usize = len('\0'.encode('unicode-internal')) - samples = ['', '1'*100] - # we need to test for both sizes, because we don't know if the string - # has been cached + # each tuple contains a string and its expected character size + # don't put any static strings here, as they may contain + # wchar_t or UTF-8 representations + samples = ['1'*100, '\xff'*50, + '\u0100'*40, '\uffff'*100, + '\U00010000'*30, '\U0010ffff'*100] + asciifields = h + "PPiP" + compactfields = asciifields + "PPP" + unicodefields = compactfields + "P" for s in samples: - basicsize = size(h + 'PPPiP') + usize * (len(s) + 1) - check(s, basicsize) + maxchar = ord(max(s)) + if maxchar < 128: + L = size(asciifields) + len(s) + 1 + elif maxchar < 256: + L = size(compactfields) + len(s) + 1 + elif maxchar < 65536: + L = size(compactfields) + 2*(len(s) + 1) + else: + L = size(compactfields) + 4*(len(s) + 1) + check(s, L) + # verify that the UTF-8 size is accounted for + s = chr(0x4000) # 4 bytes canonical representation + check(s, size(compactfields) + 4) + try: + # FIXME: codecs.lookup(str) calls encoding.search_function() which + # calls __import__ using str in the module name. __import__ encodes + # the module name to the file system encoding (which is the locale + # encoding), so test_sys fails if the locale encoding is not UTF-8. + codecs.lookup(s) # produces 4 bytes UTF-8 + except LookupError: + check(s, size(compactfields) + 4 + 4) + # TODO: add check that forces the presence of wchar_t representation + # TODO: add check that forces layout of unicodefields # weakref import weakref check(weakref.ref(int), size(h + '2Pl2P')) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1583,16 +1583,32 @@ self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize) def test_raiseMemError(self): - # Ensure that the freelist contains a consistent object, even - # when a string allocation fails with a MemoryError. - # This used to crash the interpreter, - # or leak references when the number was smaller. - charwidth = 4 if sys.maxunicode >= 0x10000 else 2 - # Note: sys.maxsize is half of the actual max allocation because of - # the signedness of Py_ssize_t. - alloc = lambda: "a" * (sys.maxsize // charwidth * 2) - self.assertRaises(MemoryError, alloc) - self.assertRaises(MemoryError, alloc) + if struct.calcsize('P') == 8: + # 64 bits pointers + ascii_struct_size = 64 + compact_struct_size = 88 + else: + # 32 bits pointers + ascii_struct_size = 32 + compact_struct_size = 44 + + for char in ('a', '\xe9', '\u20ac', '\U0010ffff'): + code = ord(char) + if code < 0x100: + char_size = 1 # sizeof(Py_UCS1) + struct_size = ascii_struct_size + elif code < 0x10000: + char_size = 2 # sizeof(Py_UCS2) + struct_size = compact_struct_size + else: + char_size = 4 # sizeof(Py_UCS4) + struct_size = compact_struct_size + # Note: sys.maxsize is half of the actual max allocation because of + # the signedness of Py_ssize_t. -1 because of the null character. + maxlen = ((sys.maxsize - struct_size) // char_size) - 1 + alloc = lambda: char * maxlen + self.assertRaises(MemoryError, alloc) + self.assertRaises(MemoryError, alloc) def test_format_subclass(self): class S(str): @@ -1608,10 +1624,7 @@ from ctypes import (pythonapi, py_object, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t) - if sys.maxunicode == 65535: - name = "PyUnicodeUCS2_FromFormat" - else: - name = "PyUnicodeUCS4_FromFormat" + name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) _PyUnicode_FromFormat.restype = py_object diff --git a/Makefile.pre.in b/Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -629,7 +629,6 @@ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/split.h \ $(srcdir)/Objects/stringlib/stringdefs.h \ - $(srcdir)/Objects/stringlib/string_format.h \ $(srcdir)/Objects/stringlib/transmogrify.h \ $(srcdir)/Objects/stringlib/unicodedefs.h \ $(srcdir)/Objects/stringlib/localeutil.h @@ -639,7 +638,7 @@ Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS) Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c \ - $(BYTESTR_DEPS) + $(BYTESTR_DEPS) $(srcdir)/Objects/stringlib/unicode_format.h Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h @@ -650,8 +649,7 @@ Python/ceval.o: $(OPCODETARGETS_H) Python/ceval_gil.h Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \ - $(BYTESTR_DEPS) \ - $(srcdir)/Objects/stringlib/formatter.h + $(BYTESTR_DEPS) Objects/typeobject.o: $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- PEP 393: flexible string representation. + - Issue #13012: The 'keepends' parameter to str.splitlines may now be passed as a keyword argument: "my_string.splitlines(keepends=True)". The same change also applies to bytes.splitlines and bytearray.splitlines. diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -700,12 +700,10 @@ return NULL; str = PyUnicode_FromObject(str); - if (str == NULL) + if (str == NULL || PyUnicode_READY(str) == -1) return NULL; - v = codec_tuple(PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(str), - PyUnicode_GET_SIZE(str), - errors), - PyUnicode_GET_SIZE(str)); + v = codec_tuple(PyUnicode_AsEncodedString(str, "utf-8", errors), + PyUnicode_GET_LENGTH(str)); Py_DECREF(str); return v; } diff --git a/Modules/_csv.c b/Modules/_csv.c --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -128,7 +128,7 @@ return Py_None; } else - return PyUnicode_FromUnicode((Py_UNICODE *)&c, 1); + return PyUnicode_FromOrdinal(c); } static PyObject * diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1843,11 +1843,9 @@ return NULL; } if (PyUnicode_Check(proto)) { - PyObject *v = _PyUnicode_AsDefaultEncodedString(proto); - if (!v) + proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); + if (!proto_str) goto error; - proto_str = PyBytes_AS_STRING(v); - proto_len = PyBytes_GET_SIZE(v); } else { PyErr_SetString(PyExc_TypeError, "class must define a '_type_' string attribute"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -658,13 +658,6 @@ #ifdef CTYPES_UNICODE if (PyUnicode_Check(obj)) { -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - pa->ffi_type = &ffi_type_pointer; - pa->value.p = PyUnicode_AS_UNICODE(obj); - Py_INCREF(obj); - pa->keep = obj; - return 0; -#else pa->ffi_type = &ffi_type_pointer; pa->value.p = PyUnicode_AsWideCharString(obj, NULL); if (pa->value.p == NULL) @@ -675,7 +668,6 @@ return -1; } return 0; -#endif } #endif diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -9,7 +9,6 @@ #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -17,7 +16,6 @@ PyMem_Free(p); } } -#endif /******************************************************************/ @@ -1238,32 +1236,24 @@ static PyObject * U_get(void *ptr, Py_ssize_t size) { - PyObject *result; Py_ssize_t len; - Py_UNICODE *p; + wchar_t *p; size /= sizeof(wchar_t); /* we count character units here, not bytes */ - result = PyUnicode_FromWideChar((wchar_t *)ptr, size); - if (!result) - return NULL; /* We need 'result' to be able to count the characters with wcslen, since ptr may not be NUL terminated. If the length is smaller (if it was actually NUL terminated, we construct a new one and throw away the result. */ /* chop off at the first NUL character, if any. */ - p = PyUnicode_AS_UNICODE(result); - for (len = 0; len < size; ++len) + p = (wchar_t*)ptr; + for (len = 0; len < size; ++len) { if (!p[len]) break; + } - if (len < size) { - PyObject *ob = PyUnicode_FromWideChar((wchar_t *)ptr, len); - Py_DECREF(result); - return ob; - } - return result; + return PyUnicode_FromWideChar((wchar_t *)ptr, len); } static PyObject * @@ -1401,6 +1391,9 @@ static PyObject * Z_set(void *ptr, PyObject *value, Py_ssize_t size) { + PyObject *keep; + wchar_t *buffer; + if (value == Py_None) { *(wchar_t **)ptr = NULL; Py_INCREF(value); @@ -1420,37 +1413,20 @@ "unicode string or integer address expected instead of %s instance", value->ob_type->tp_name); return NULL; - } else - Py_INCREF(value); -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - /* We can copy directly. Hm, are unicode objects always NUL - terminated in Python, internally? - */ - *(wchar_t **)ptr = (wchar_t *) PyUnicode_AS_UNICODE(value); - return value; -#else - { - /* We must create a wchar_t* buffer from the unicode object, - and keep it alive */ - PyObject *keep; - wchar_t *buffer; + } - buffer = PyUnicode_AsWideCharString(value, NULL); - if (!buffer) { - Py_DECREF(value); - return NULL; - } - keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); - if (!keep) { - Py_DECREF(value); - PyMem_Free(buffer); - return NULL; - } - *(wchar_t **)ptr = (wchar_t *)buffer; - Py_DECREF(value); - return keep; + /* We must create a wchar_t* buffer from the unicode object, + and keep it alive */ + buffer = PyUnicode_AsWideCharString(value, NULL); + if (!buffer) + return NULL; + keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); + if (!keep) { + PyMem_Free(buffer); + return NULL; } -#endif + *(wchar_t **)ptr = buffer; + return keep; } static PyObject * diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -203,8 +203,11 @@ } else if(PyBytes_Check(obj) && (PyBytes_Size(obj) == 1)) { *ch = (chtype) *PyBytes_AsString(obj); - } else if (PyUnicode_Check(obj) && PyUnicode_GetSize(obj) == 1) { - *ch = (chtype) *PyUnicode_AS_UNICODE(obj); + } else if (PyUnicode_Check(obj) && PyUnicode_GET_LENGTH(obj) == 1) { + Py_UCS4 ucs = PyUnicode_READ(PyUnicode_KIND(obj), + PyUnicode_DATA(obj), + 0); + *ch = (chtype)ucs; } else { return 0; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -985,9 +985,8 @@ if (tzinfo == Py_None) return repr; /* Get rid of the trailing ')'. */ - assert(PyUnicode_AS_UNICODE(repr)[PyUnicode_GET_SIZE(repr)-1] == ')'); - temp = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(repr), - PyUnicode_GET_SIZE(repr) - 1); + assert(PyUnicode_READ_CHAR(repr, PyUnicode_GET_LENGTH(repr)-1) == ')'); + temp = PyUnicode_Substring(repr, 0, PyUnicode_GET_LENGTH(repr) - 1); Py_DECREF(repr); if (temp == NULL) return NULL; @@ -4214,9 +4213,9 @@ datetime_strptime(PyObject *cls, PyObject *args) { static PyObject *module = NULL; - const Py_UNICODE *string, *format; - - if (!PyArg_ParseTuple(args, "uu:strptime", &string, &format)) + PyObject *string, *format; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; if (module == NULL) { @@ -4224,7 +4223,7 @@ if (module == NULL) return NULL; } - return PyObject_CallMethod(module, "_strptime_datetime", "Ouu", + return PyObject_CallMethod(module, "_strptime_datetime", "OOO", cls, string, format); } diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -212,6 +212,7 @@ { dbmobject *dp = (dbmobject *)self; datum key, val; + Py_ssize_t size; if ((dp)->di_dbm == NULL) { PyErr_SetString(DbmError, @@ -219,8 +220,9 @@ return -1; } if (PyUnicode_Check(arg)) { - arg = _PyUnicode_AsDefaultEncodedString(arg); - if (arg == NULL) + key.dptr = PyUnicode_AsUTF8AndSize(arg, &size); + key.dsize = size; + if (key.dptr == NULL) return -1; } if (!PyBytes_Check(arg)) { @@ -229,8 +231,10 @@ arg->ob_type->tp_name); return -1; } - key.dptr = PyBytes_AS_STRING(arg); - key.dsize = PyBytes_GET_SIZE(arg); + else { + key.dptr = PyBytes_AS_STRING(arg); + key.dsize = PyBytes_GET_SIZE(arg); + } val = dbm_fetch(dp->di_dbm, key); return val.dptr != NULL; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -723,13 +723,16 @@ (ch == '/' || ch == '*' || ch == '[' || ch == '@' || ch == '.') if (PyUnicode_Check(tag)) { - Py_UNICODE *p = PyUnicode_AS_UNICODE(tag); - for (i = 0; i < PyUnicode_GET_SIZE(tag); i++) { - if (p[i] == '{') + const Py_ssize_t len = PyUnicode_GET_LENGTH(tag); + void *data = PyUnicode_DATA(tag); + unsigned int kind = PyUnicode_KIND(tag); + for (i = 0; i < len; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch == '{') check = 0; - else if (p[i] == '}') + else if (ch == '}') check = 1; - else if (check && PATHCHAR(p[i])) + else if (check && PATHCHAR(ch)) return 1; } return 0; @@ -2401,9 +2404,10 @@ XML_Encoding *info) { PyObject* u; - Py_UNICODE* p; unsigned char s[256]; int i; + void *data; + unsigned int kind; memset(info, 0, sizeof(XML_Encoding)); @@ -2413,17 +2417,20 @@ u = PyUnicode_Decode((char*) s, 256, name, "replace"); if (!u) return XML_STATUS_ERROR; - - if (PyUnicode_GET_SIZE(u) != 256) { + if (PyUnicode_READY(u)) + return XML_STATUS_ERROR; + + if (PyUnicode_GET_LENGTH(u) != 256) { Py_DECREF(u); return XML_STATUS_ERROR; } - p = PyUnicode_AS_UNICODE(u); - + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); for (i = 0; i < 256; i++) { - if (p[i] != Py_UNICODE_REPLACEMENT_CHARACTER) - info->map[i] = p[i]; + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch != Py_UNICODE_REPLACEMENT_CHARACTER) + info->map[i] = ch; else info->map[i] = -1; } diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -55,7 +55,7 @@ Otherwise, the function will scan further and return garbage. */ extern Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed); + int kind, char *start, char *end, Py_ssize_t *consumed); #define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */ diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -9,7 +9,7 @@ typedef struct { PyObject_HEAD - Py_UNICODE *buf; + Py_UCS4 *buf; Py_ssize_t pos; Py_ssize_t string_size; size_t buf_size; @@ -21,7 +21,7 @@ PyObject *decoder; PyObject *readnl; PyObject *writenl; - + PyObject *dict; PyObject *weakreflist; } stringio; @@ -56,7 +56,7 @@ /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ size_t alloc = self->buf_size; - Py_UNICODE *new_buf = NULL; + Py_UCS4 *new_buf = NULL; assert(self->buf != NULL); @@ -84,10 +84,9 @@ alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(Py_UNICODE)) + if (alloc > PY_SIZE_MAX / sizeof(Py_UCS4)) goto overflow; - new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf, - alloc * sizeof(Py_UNICODE)); + new_buf = (Py_UCS4 *)PyMem_Realloc(self->buf, alloc * sizeof(Py_UCS4)); if (new_buf == NULL) { PyErr_NoMemory(); return -1; @@ -108,9 +107,9 @@ static Py_ssize_t write_str(stringio *self, PyObject *obj) { - Py_UNICODE *str; Py_ssize_t len; PyObject *decoded = NULL; + assert(self->buf != NULL); assert(self->pos >= 0); @@ -132,8 +131,7 @@ return -1; assert(PyUnicode_Check(decoded)); - str = PyUnicode_AS_UNICODE(decoded); - len = PyUnicode_GET_SIZE(decoded); + len = PyUnicode_GET_LENGTH(decoded); assert(len >= 0); @@ -161,18 +159,21 @@ */ memset(self->buf + self->string_size, '\0', - (self->pos - self->string_size) * sizeof(Py_UNICODE)); + (self->pos - self->string_size) * sizeof(Py_UCS4)); } /* Copy the data to the internal buffer, overwriting some of the existing data if self->pos < self->string_size. */ - memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE)); - self->pos += len; + if (!PyUnicode_AsUCS4(decoded, + self->buf + self->pos, + self->buf_size - self->pos, + 0)) + goto fail; /* Set the new length of the internal string if it has changed. */ - if (self->string_size < self->pos) { + self->pos += len; + if (self->string_size < self->pos) self->string_size = self->pos; - } Py_DECREF(decoded); return 0; @@ -190,7 +191,8 @@ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); - return PyUnicode_FromUnicode(self->buf, self->string_size); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, self->buf, + self->string_size); } PyDoc_STRVAR(stringio_tell_doc, @@ -214,7 +216,7 @@ stringio_read(stringio *self, PyObject *args) { Py_ssize_t size, n; - Py_UNICODE *output; + Py_UCS4 *output; PyObject *arg = Py_None; CHECK_INITIALIZED(self); @@ -247,19 +249,19 @@ output = self->buf + self->pos; self->pos += size; - return PyUnicode_FromUnicode(output, size); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, size); } /* Internal helper, used by stringio_readline and stringio_iternext */ static PyObject * _stringio_readline(stringio *self, Py_ssize_t limit) { - Py_UNICODE *start, *end, old_char; + Py_UCS4 *start, *end, old_char; Py_ssize_t len, consumed; /* In case of overseek, return the empty string */ if (self->pos >= self->string_size) - return PyUnicode_FromString(""); + return PyUnicode_New(0, 0); start = self->buf + self->pos; if (limit < 0 || limit > self->string_size - self->pos) @@ -270,14 +272,14 @@ *end = '\0'; len = _PyIO_find_line_ending( self->readtranslate, self->readuniversal, self->readnl, - start, end, &consumed); + PyUnicode_4BYTE_KIND, (char*)start, (char*)end, &consumed); *end = old_char; /* If we haven't found any line ending, we just return everything (`consumed` is ignored). */ if (len < 0) len = limit; self->pos += len; - return PyUnicode_FromUnicode(start, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, start, len); } PyDoc_STRVAR(stringio_readline_doc, @@ -462,8 +464,10 @@ Py_TYPE(obj)->tp_name); return NULL; } + if (PyUnicode_READY(obj)) + return NULL; CHECK_CLOSED(self); - size = PyUnicode_GET_SIZE(obj); + size = PyUnicode_GET_LENGTH(obj); if (size > 0 && write_str(self, obj) < 0) return NULL; @@ -535,7 +539,7 @@ /* tp_alloc initializes all the fields to zero. So we don't have to initialize them here. */ - self->buf = (Py_UNICODE *)PyMem_Malloc(0); + self->buf = (Py_UCS4 *)PyMem_Malloc(0); if (self->buf == NULL) { Py_DECREF(self); return PyErr_NoMemory(); @@ -747,11 +751,22 @@ once by __init__. So we do not take any chance and replace object's buffer completely. */ { - Py_UNICODE *buf = PyUnicode_AS_UNICODE(PyTuple_GET_ITEM(state, 0)); - Py_ssize_t bufsize = PyUnicode_GET_SIZE(PyTuple_GET_ITEM(state, 0)); - if (resize_buffer(self, bufsize) < 0) + PyObject *item; + Py_UCS4 *buf; + Py_ssize_t bufsize; + + item = PyTuple_GET_ITEM(state, 0); + buf = PyUnicode_AsUCS4Copy(item); + if (buf == NULL) return NULL; - memcpy(self->buf, buf, bufsize * sizeof(Py_UNICODE)); + bufsize = PyUnicode_GET_LENGTH(item); + + if (resize_buffer(self, bufsize) < 0) { + PyMem_Free(buf); + return NULL; + } + memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); + PyMem_Free(buf); self->string_size = bufsize; } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -274,18 +274,28 @@ goto error; } - output_len = PyUnicode_GET_SIZE(output); + if (PyUnicode_READY(output) == -1) + goto error; + + output_len = PyUnicode_GET_LENGTH(output); if (self->pendingcr && (final || output_len > 0)) { - Py_UNICODE *out; - PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1); + /* Prefix output with CR */ + int kind; + PyObject *modified; + char *out; + + modified = PyUnicode_New(output_len + 1, + PyUnicode_MAX_CHAR_VALUE(output)); if (modified == NULL) goto error; - out = PyUnicode_AS_UNICODE(modified); - out[0] = '\r'; - memcpy(out + 1, PyUnicode_AS_UNICODE(output), - output_len * sizeof(Py_UNICODE)); + kind = PyUnicode_KIND(modified); + out = PyUnicode_DATA(modified); + PyUnicode_WRITE(kind, PyUnicode_DATA(modified), 0, '\r'); + memcpy(out + PyUnicode_KIND_SIZE(kind, 1), + PyUnicode_DATA(output), + PyUnicode_KIND_SIZE(kind, output_len)); Py_DECREF(output); - output = modified; + output = modified; /* output remains ready */ self->pendingcr = 0; output_len++; } @@ -295,21 +305,13 @@ */ if (!final) { if (output_len > 0 - && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') { - - if (Py_REFCNT(output) == 1) { - if (PyUnicode_Resize(&output, output_len - 1) < 0) - goto error; - } - else { - PyObject *modified = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(output), - output_len - 1); - if (modified == NULL) - goto error; - Py_DECREF(output); - output = modified; - } + && PyUnicode_READ_CHAR(output, output_len - 1) == '\r') + { + PyObject *modified = PyUnicode_Substring(output, 0, output_len -1); + if (modified == NULL) + goto error; + Py_DECREF(output); + output = modified; self->pendingcr = 1; } } @@ -317,13 +319,15 @@ /* Record which newlines are read and do newline translation if desired, all in one pass. */ { - Py_UNICODE *in_str; + void *in_str; Py_ssize_t len; int seennl = self->seennl; int only_lf = 0; + int kind; - in_str = PyUnicode_AS_UNICODE(output); - len = PyUnicode_GET_SIZE(output); + in_str = PyUnicode_DATA(output); + len = PyUnicode_GET_LENGTH(output); + kind = PyUnicode_KIND(output); if (len == 0) return output; @@ -332,7 +336,7 @@ for the \r *byte* with the libc's optimized memchr. */ if (seennl == SEEN_LF || seennl == 0) { - only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL); + only_lf = (memchr(in_str, '\r', PyUnicode_KIND_SIZE(kind, len)) == NULL); } if (only_lf) { @@ -340,21 +344,19 @@ (there's nothing else to be done, even when in translation mode) */ if (seennl == 0 && - memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) { - Py_UNICODE *s, *end; - s = in_str; - end = in_str + len; + memchr(in_str, '\n', PyUnicode_KIND_SIZE(kind, len)) != NULL) { + Py_ssize_t i = 0; for (;;) { Py_UNICODE c; /* Fast loop for non-control characters */ - while (*s > '\n') - s++; - c = *s++; + while (PyUnicode_READ(kind, in_str, i) > '\n') + i++; + c = PyUnicode_READ(kind, in_str, i++); if (c == '\n') { seennl |= SEEN_LF; break; } - if (s > end) + if (i >= len) break; } } @@ -362,29 +364,27 @@ need translating */ } else if (!self->translate) { - Py_UNICODE *s, *end; + Py_ssize_t i = 0; /* We have already seen all newline types, no need to scan again */ if (seennl == SEEN_ALL) goto endscan; - s = in_str; - end = in_str + len; for (;;) { - Py_UNICODE c; + Py_UCS4 c; /* Fast loop for non-control characters */ - while (*s > '\r') - s++; - c = *s++; + while (PyUnicode_READ(kind, in_str, i) > '\r') + i++; + c = PyUnicode_READ(kind, in_str, i++); if (c == '\n') seennl |= SEEN_LF; else if (c == '\r') { - if (*s == '\n') { + if (PyUnicode_READ(kind, in_str, i) == '\n') { seennl |= SEEN_CRLF; - s++; + i++; } else seennl |= SEEN_CR; } - if (s > end) + if (i >= len) break; if (seennl == SEEN_ALL) break; @@ -393,61 +393,50 @@ ; } else { - PyObject *translated = NULL; - Py_UNICODE *out_str; - Py_UNICODE *in, *out, *end; - if (Py_REFCNT(output) != 1) { - /* We could try to optimize this so that we only do a copy - when there is something to translate. On the other hand, - most decoders should only output non-shared strings, i.e. - translation is done in place. */ - translated = PyUnicode_FromUnicode(NULL, len); - if (translated == NULL) - goto error; - assert(Py_REFCNT(translated) == 1); - memcpy(PyUnicode_AS_UNICODE(translated), - PyUnicode_AS_UNICODE(output), - len * sizeof(Py_UNICODE)); + void *translated; + int kind = PyUnicode_KIND(output); + void *in_str = PyUnicode_DATA(output); + Py_ssize_t in, out; + /* XXX: Previous in-place translation here is disabled as + resizing is not possible anymore */ + /* We could try to optimize this so that we only do a copy + when there is something to translate. On the other hand, + we already know there is a \r byte, so chances are high + that something needs to be done. */ + translated = PyMem_Malloc(PyUnicode_KIND_SIZE(kind, len)); + if (translated == NULL) { + PyErr_NoMemory(); + goto error; } - else { - translated = output; - } - out_str = PyUnicode_AS_UNICODE(translated); - in = in_str; - out = out_str; - end = in_str + len; + in = out = 0; for (;;) { - Py_UNICODE c; + Py_UCS4 c; /* Fast loop for non-control characters */ - while ((c = *in++) > '\r') - *out++ = c; + while ((c = PyUnicode_READ(kind, in_str, in++)) > '\r') + PyUnicode_WRITE(kind, translated, out++, c); if (c == '\n') { - *out++ = c; + PyUnicode_WRITE(kind, translated, out++, c); seennl |= SEEN_LF; continue; } if (c == '\r') { - if (*in == '\n') { + if (PyUnicode_READ(kind, in_str, in) == '\n') { in++; seennl |= SEEN_CRLF; } else seennl |= SEEN_CR; - *out++ = '\n'; + PyUnicode_WRITE(kind, translated, out++, '\n'); continue; } - if (in > end) + if (in > len) break; - *out++ = c; + PyUnicode_WRITE(kind, translated, out++, c); } - if (translated != output) { - Py_DECREF(output); - output = translated; - } - if (out - out_str != len) { - if (PyUnicode_Resize(&output, out - out_str) < 0) - goto error; - } + Py_DECREF(output); + output = PyUnicode_FromKindAndData(kind, translated, out); + if (!output) + goto error; } self->seennl |= seennl; } @@ -705,9 +694,7 @@ static PyObject * ascii_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsASCIIString(text, PyBytes_AS_STRING(self->errors)); } static PyObject * @@ -777,17 +764,13 @@ static PyObject * utf8_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsUTF8String(text, PyBytes_AS_STRING(self->errors)); } static PyObject * latin1_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsLatin1String(text, PyBytes_AS_STRING(self->errors)); } /* Map normalized encoding names onto the specialized encoding funcs */ @@ -1213,18 +1196,6 @@ return buffer; } -Py_LOCAL_INLINE(const Py_UNICODE *) -findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch) -{ - /* like wcschr, but doesn't stop at NULL characters */ - while (size-- > 0) { - if (*s == ch) - return s; - s++; - } - return NULL; -} - /* Flush the internal write buffer. This doesn't explicitly flush the underlying buffered object, though. */ static int @@ -1269,6 +1240,9 @@ return NULL; } + if (PyUnicode_READY(text) == -1) + return NULL; + CHECK_CLOSED(self); if (self->encoder == NULL) @@ -1276,11 +1250,10 @@ Py_INCREF(text); - textlen = PyUnicode_GetSize(text); + textlen = PyUnicode_GET_LENGTH(text); if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) - if (findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\n')) + if (PyUnicode_FindChar(text, '\n', 0, PyUnicode_GET_LENGTH(text), 1) != -1) haslf = 1; if (haslf && self->writetranslate && self->writenl != NULL) { @@ -1296,8 +1269,7 @@ needflush = 1; else if (self->line_buffering && (haslf || - findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\r'))) + PyUnicode_FindChar(text, '\r', 0, PyUnicode_GET_LENGTH(text), 1) != -1)) needflush = 1; /* XXX What if we were just reading? */ @@ -1369,7 +1341,8 @@ if (self->decoded_chars == NULL) return PyUnicode_FromStringAndSize(NULL, 0); - avail = (PyUnicode_GET_SIZE(self->decoded_chars) + /* decoded_chars is guaranteed to be "ready". */ + avail = (PyUnicode_GET_LENGTH(self->decoded_chars) - self->decoded_chars_used); assert(avail >= 0); @@ -1378,9 +1351,9 @@ n = avail; if (self->decoded_chars_used > 0 || n < avail) { - chars = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(self->decoded_chars) - + self->decoded_chars_used, n); + chars = PyUnicode_Substring(self->decoded_chars, + self->decoded_chars_used, + self->decoded_chars_used + n); if (chars == NULL) return NULL; } @@ -1464,8 +1437,10 @@ /* TODO sanity check: isinstance(decoded_chars, unicode) */ if (decoded_chars == NULL) goto fail; + if (PyUnicode_READY(decoded_chars) == -1) + goto fail; textiowrapper_set_decoded_chars(self, decoded_chars); - nchars = PyUnicode_GET_SIZE(decoded_chars); + nchars = PyUnicode_GET_LENGTH(decoded_chars); if (nchars > 0) self->b2cratio = (double) nbytes / nchars; else @@ -1553,7 +1528,9 @@ result = textiowrapper_get_decoded_chars(self, n); if (result == NULL) goto fail; - remaining -= PyUnicode_GET_SIZE(result); + if (PyUnicode_READY(result) == -1) + goto fail; + remaining -= PyUnicode_GET_LENGTH(result); /* Keep reading chunks until we have n characters to return */ while (remaining > 0) { @@ -1573,7 +1550,7 @@ result = textiowrapper_get_decoded_chars(self, remaining); if (result == NULL) goto fail; - remaining -= PyUnicode_GET_SIZE(result); + remaining -= PyUnicode_GET_LENGTH(result); } if (chunks != NULL) { if (result != NULL && PyList_Append(chunks, result) < 0) @@ -1596,33 +1573,34 @@ /* NOTE: `end` must point to the real end of the Py_UNICODE storage, that is to the NUL character. Otherwise the function will produce incorrect results. */ -static Py_UNICODE * -find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch) +static char * +find_control_char(int kind, char *s, char *end, Py_UCS4 ch) { - Py_UNICODE *s = start; + int size = PyUnicode_KIND_SIZE(kind, 1); for (;;) { - while (*s > ch) - s++; - if (*s == ch) + while (PyUnicode_READ(kind, s, 0) > ch) + s += size; + if (PyUnicode_READ(kind, s, 0) == ch) return s; if (s == end) return NULL; - s++; + s += size; } } Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed) + int kind, char *start, char *end, Py_ssize_t *consumed) { - Py_ssize_t len = end - start; + int size = PyUnicode_KIND_SIZE(kind, 1); + Py_ssize_t len = ((char*)end - (char*)start)/size; if (translated) { /* Newlines are already translated, only search for \n */ - Py_UNICODE *pos = find_control_char(start, end, '\n'); + char *pos = find_control_char(kind, start, end, '\n'); if (pos != NULL) - return pos - start + 1; + return (pos - start)/size + 1; else { *consumed = len; return -1; @@ -1632,63 +1610,66 @@ /* Universal newline search. Find any of \r, \r\n, \n * The decoder ensures that \r\n are not split in two pieces */ - Py_UNICODE *s = start; + char *s = start; for (;;) { - Py_UNICODE ch; + Py_UCS4 ch; /* Fast path for non-control chars. The loop always ends since the Py_UNICODE storage is NUL-terminated. */ - while (*s > '\r') - s++; + while (PyUnicode_READ(kind, s, 0) > '\r') + s += size; if (s >= end) { *consumed = len; return -1; } - ch = *s++; + ch = PyUnicode_READ(kind, s, 0); + s += size; if (ch == '\n') - return s - start; + return (s - start)/size; if (ch == '\r') { - if (*s == '\n') - return s - start + 1; + if (PyUnicode_READ(kind, s, 0) == '\n') + return (s - start)/size + 1; else - return s - start; + return (s - start)/size; } } } else { /* Non-universal mode. */ - Py_ssize_t readnl_len = PyUnicode_GET_SIZE(readnl); - Py_UNICODE *nl = PyUnicode_AS_UNICODE(readnl); + Py_ssize_t readnl_len = PyUnicode_GET_LENGTH(readnl); + char *nl = PyUnicode_DATA(readnl); + /* Assume that readnl is an ASCII character. */ + assert(PyUnicode_KIND(readnl) == PyUnicode_1BYTE_KIND); if (readnl_len == 1) { - Py_UNICODE *pos = find_control_char(start, end, nl[0]); + char *pos = find_control_char(kind, start, end, nl[0]); if (pos != NULL) - return pos - start + 1; + return (pos - start)/size + 1; *consumed = len; return -1; } else { - Py_UNICODE *s = start; - Py_UNICODE *e = end - readnl_len + 1; - Py_UNICODE *pos; + char *s = start; + char *e = end - (readnl_len - 1)*size; + char *pos; if (e < s) e = s; while (s < e) { Py_ssize_t i; - Py_UNICODE *pos = find_control_char(s, end, nl[0]); + char *pos = find_control_char(kind, s, end, nl[0]); if (pos == NULL || pos >= e) break; for (i = 1; i < readnl_len; i++) { - if (pos[i] != nl[i]) + if (PyUnicode_READ(kind, pos, i) != nl[i]) break; } if (i == readnl_len) - return pos - start + readnl_len; - s = pos + 1; + return (pos - start)/size + readnl_len; + s = pos + size; } - pos = find_control_char(e, end, nl[0]); + pos = find_control_char(kind, e, end, nl[0]); if (pos == NULL) *consumed = len; else - *consumed = pos - start; + *consumed = (pos - start)/size; return -1; } } @@ -1709,14 +1690,15 @@ chunked = 0; while (1) { - Py_UNICODE *ptr; + char *ptr; Py_ssize_t line_len; + int kind; Py_ssize_t consumed = 0; /* First, get some data if necessary */ res = 1; while (!self->decoded_chars || - !PyUnicode_GET_SIZE(self->decoded_chars)) { + !PyUnicode_GET_LENGTH(self->decoded_chars)) { res = textiowrapper_read_chunk(self); if (res < 0) goto error; @@ -1741,18 +1723,24 @@ assert(self->decoded_chars_used == 0); line = PyUnicode_Concat(remaining, self->decoded_chars); start = 0; - offset_to_buffer = PyUnicode_GET_SIZE(remaining); + offset_to_buffer = PyUnicode_GET_LENGTH(remaining); Py_CLEAR(remaining); if (line == NULL) goto error; + if (PyUnicode_READY(line) == -1) + goto error; } - ptr = PyUnicode_AS_UNICODE(line); - line_len = PyUnicode_GET_SIZE(line); + ptr = PyUnicode_DATA(line); + line_len = PyUnicode_GET_LENGTH(line); + kind = PyUnicode_KIND(line); endpos = _PyIO_find_line_ending( self->readtranslate, self->readuniversal, self->readnl, - ptr + start, ptr + line_len, &consumed); + kind, + ptr + PyUnicode_KIND_SIZE(kind, start), + ptr + PyUnicode_KIND_SIZE(kind, line_len), + &consumed); if (endpos >= 0) { endpos += start; if (limit >= 0 && (endpos - start) + chunked >= limit) @@ -1776,21 +1764,20 @@ if (chunks == NULL) goto error; } - s = PyUnicode_FromUnicode(ptr + start, endpos - start); + s = PyUnicode_Substring(line, start, endpos); if (s == NULL) goto error; if (PyList_Append(chunks, s) < 0) { Py_DECREF(s); goto error; } - chunked += PyUnicode_GET_SIZE(s); + chunked += PyUnicode_GET_LENGTH(s); Py_DECREF(s); } /* There may be some remaining bytes we'll have to prepend to the next chunk of data */ if (endpos < line_len) { - remaining = PyUnicode_FromUnicode( - ptr + endpos, line_len - endpos); + remaining = PyUnicode_Substring(line, endpos, line_len); if (remaining == NULL) goto error; } @@ -1802,19 +1789,12 @@ if (line != NULL) { /* Our line ends in the current buffer */ self->decoded_chars_used = endpos - offset_to_buffer; - if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) { - if (start == 0 && Py_REFCNT(line) == 1) { - if (PyUnicode_Resize(&line, endpos) < 0) - goto error; - } - else { - PyObject *s = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(line) + start, endpos - start); - Py_CLEAR(line); - if (s == NULL) - goto error; - line = s; - } + if (start > 0 || endpos < PyUnicode_GET_LENGTH(line)) { + PyObject *s = PyUnicode_Substring(line, start, endpos); + Py_CLEAR(line); + if (s == NULL) + goto error; + line = s; } } if (remaining != NULL) { @@ -1828,16 +1808,20 @@ Py_CLEAR(remaining); } if (chunks != NULL) { - if (line != NULL && PyList_Append(chunks, line) < 0) - goto error; - Py_CLEAR(line); + if (line != NULL) { + if (PyList_Append(chunks, line) < 0) + goto error; + Py_DECREF(line); + } line = PyUnicode_Join(_PyIO_empty_str, chunks); if (line == NULL) goto error; - Py_DECREF(chunks); + Py_CLEAR(chunks); } - if (line == NULL) - line = PyUnicode_FromStringAndSize(NULL, 0); + if (line == NULL) { + Py_INCREF(_PyIO_empty_str); + line = _PyIO_empty_str; + } return line; @@ -2128,6 +2112,10 @@ if (decoded == NULL) goto fail; + if (PyUnicode_READY(decoded) == -1) { + Py_DECREF(decoded); + goto fail; + } textiowrapper_set_decoded_chars(self, decoded); @@ -2250,7 +2238,7 @@ if (_decoded == NULL) \ goto fail; \ assert (PyUnicode_Check(_decoded)); \ - res = PyUnicode_GET_SIZE(_decoded); \ + res = PyUnicode_GET_LENGTH(_decoded); \ Py_DECREF(_decoded); \ } while (0) @@ -2333,7 +2321,7 @@ if (decoded == NULL) goto fail; assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); + chars_decoded += PyUnicode_GET_LENGTH(decoded); Py_DECREF(decoded); cookie.need_eof = 1; @@ -2559,10 +2547,10 @@ } } - if (line == NULL) + if (line == NULL || PyUnicode_READY(line) == -1) return NULL; - if (PyUnicode_GET_SIZE(line) == 0) { + if (PyUnicode_GET_LENGTH(line) == 0) { /* Reached EOF or would have blocked */ Py_DECREF(line); Py_CLEAR(self->snapshot); diff --git a/Modules/_json.c b/Modules/_json.c --- a/Modules/_json.c +++ b/Modules/_json.c @@ -238,13 +238,6 @@ #define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) -#define MIN_EXPANSION 6 -#ifdef Py_UNICODE_WIDE -#define MAX_EXPANSION (2 * MIN_EXPANSION) -#else -#define MAX_EXPANSION MIN_EXPANSION -#endif - static int _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) { @@ -263,7 +256,7 @@ } static Py_ssize_t -ascii_escape_unichar(Py_UNICODE c, Py_UNICODE *output, Py_ssize_t chars) +ascii_escape_unichar(Py_UCS4 c, unsigned char *output, Py_ssize_t chars) { /* Escape unicode code point c to ASCII escape sequences in char *output. output must have at least 12 bytes unused to @@ -278,10 +271,9 @@ case '\r': output[chars++] = 'r'; break; case '\t': output[chars++] = 't'; break; default: -#ifdef Py_UNICODE_WIDE if (c >= 0x10000) { /* UTF-16 surrogate pair */ - Py_UNICODE v = c - 0x10000; + Py_UCS4 v = c - 0x10000; c = 0xd800 | ((v >> 10) & 0x3ff); output[chars++] = 'u'; output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; @@ -291,7 +283,6 @@ c = 0xdc00 | (v & 0x3ff); output[chars++] = '\\'; } -#endif output[chars++] = 'u'; output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; @@ -308,54 +299,52 @@ Py_ssize_t i; Py_ssize_t input_chars; Py_ssize_t output_size; - Py_ssize_t max_output_size; Py_ssize_t chars; PyObject *rval; - Py_UNICODE *output; - Py_UNICODE *input_unicode; + void *input; + unsigned char *output; + int kind; - input_chars = PyUnicode_GET_SIZE(pystr); - input_unicode = PyUnicode_AS_UNICODE(pystr); + if (PyUnicode_READY(pystr) == -1) + return NULL; - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - max_output_size = 2 + (input_chars * MAX_EXPANSION); - rval = PyUnicode_FromStringAndSize(NULL, output_size); + input_chars = PyUnicode_GET_LENGTH(pystr); + input = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + + /* Compute the output size */ + for (i = 0, output_size = 2; i < input_chars; i++) { + Py_UCS4 c = PyUnicode_READ(kind, input, i); + if (S_CHAR(c)) + output_size++; + else { + switch(c) { + case '\\': case '"': case '\b': case '\f': + case '\n': case '\r': case '\t': + output_size += 2; break; + default: + output_size += c >= 0x10000 ? 12 : 6; + } + } + } + + rval = PyUnicode_New(output_size, 127); if (rval == NULL) { return NULL; } - output = PyUnicode_AS_UNICODE(rval); + output = PyUnicode_1BYTE_DATA(rval); chars = 0; output[chars++] = '"'; for (i = 0; i < input_chars; i++) { - Py_UNICODE c = input_unicode[i]; + Py_UCS4 c = PyUnicode_READ(kind, input, i); if (S_CHAR(c)) { output[chars++] = c; } else { chars = ascii_escape_unichar(c, output, chars); } - if (output_size - chars < (1 + MAX_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - Py_ssize_t new_output_size = output_size * 2; - /* This is an upper bound */ - if (new_output_size > max_output_size) { - new_output_size = max_output_size; - } - /* Make sure that the output size changed before resizing */ - if (new_output_size != output_size) { - output_size = new_output_size; - if (PyUnicode_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyUnicode_AS_UNICODE(rval); - } - } } output[chars++] = '"'; - if (PyUnicode_Resize(&rval, chars) == -1) { - return NULL; - } return rval; } @@ -436,22 +425,30 @@ Return value is a new PyUnicode */ PyObject *rval = NULL; - Py_ssize_t len = PyUnicode_GET_SIZE(pystr); + Py_ssize_t len; Py_ssize_t begin = end - 1; Py_ssize_t next /* = begin */; - const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); + const void *buf; + int kind; PyObject *chunks = NULL; PyObject *chunk = NULL; + if (PyUnicode_READY(pystr) == -1) + return 0; + + len = PyUnicode_GET_LENGTH(pystr); + buf = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + if (end < 0 || len <= end) { PyErr_SetString(PyExc_ValueError, "end is out of bounds"); goto bail; } while (1) { /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; + Py_UCS4 c = 0; for (next = end; next < len; next++) { - c = buf[next]; + c = PyUnicode_READ(kind, buf, next); if (c == '"' || c == '\\') { break; } @@ -467,7 +464,10 @@ /* Pick up this chunk if it's not zero length */ if (next != end) { APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&buf[end], next - end); + chunk = PyUnicode_FromKindAndData( + kind, + (char*)buf + PyUnicode_KIND_SIZE(kind, end), + next - end); if (chunk == NULL) { goto bail; } @@ -481,7 +481,7 @@ raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; } - c = buf[next]; + c = PyUnicode_READ(kind, buf, next); if (c != 'u') { /* Non-unicode backslash escapes */ end = next + 1; @@ -511,7 +511,7 @@ } /* Decode 4 hex digits */ for (; next < end; next++) { - Py_UNICODE digit = buf[next]; + Py_UCS4 digit = PyUnicode_READ(kind, buf, next); c <<= 4; switch (digit) { case '0': case '1': case '2': case '3': case '4': @@ -528,22 +528,22 @@ goto bail; } } -#ifdef Py_UNICODE_WIDE /* Surrogate pair */ if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; + Py_UCS4 c2 = 0; if (end + 6 >= len) { raise_errmsg("Unpaired high surrogate", pystr, end - 5); goto bail; } - if (buf[next++] != '\\' || buf[next++] != 'u') { + if (PyUnicode_READ(kind, buf, next++) != '\\' || + PyUnicode_READ(kind, buf, next++) != 'u') { raise_errmsg("Unpaired high surrogate", pystr, end - 5); goto bail; } end += 6; /* Decode 4 hex digits */ for (; next < end; next++) { - Py_UNICODE digit = buf[next]; + Py_UCS4 digit = PyUnicode_READ(kind, buf, next); c2 <<= 4; switch (digit) { case '0': case '1': case '2': case '3': case '4': @@ -570,10 +570,9 @@ raise_errmsg("Unpaired low surrogate", pystr, end - 5); goto bail; } -#endif } APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&c, 1); + chunk = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &c, 1); if (chunk == NULL) { goto bail; } @@ -711,8 +710,9 @@ Returns a new PyObject (usually a dict, but object_hook can change that) */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; PyObject *val = NULL; PyObject *rval = NULL; PyObject *key = NULL; @@ -720,6 +720,13 @@ int has_pairs_hook = (s->object_pairs_hook != Py_None); Py_ssize_t next_idx; + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; + if (has_pairs_hook) rval = PyList_New(0); else @@ -728,15 +735,15 @@ return NULL; /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind,str, idx))) idx++; /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { while (idx <= end_idx) { PyObject *memokey; /* read key */ - if (str[idx] != '"') { + if (PyUnicode_READ(kind, str, idx) != '"') { raise_errmsg("Expecting property name", pystr, idx); goto bail; } @@ -756,13 +763,13 @@ idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { raise_errmsg("Expecting : delimiter", pystr, idx); goto bail; } idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* read any JSON term */ val = scan_once_unicode(s, pystr, idx, &next_idx); @@ -790,26 +797,26 @@ idx = next_idx; /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* bail if the object is closed or we didn't get the , delimiter */ if (idx > end_idx) break; - if (str[idx] == '}') { + if (PyUnicode_READ(kind, str, idx) == '}') { break; } - else if (str[idx] != ',') { + else if (PyUnicode_READ(kind, str, idx) != ',') { raise_errmsg("Expecting , delimiter", pystr, idx); goto bail; } idx++; /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; } } /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { raise_errmsg("Expecting object", pystr, end_idx); goto bail; } @@ -845,19 +852,27 @@ Returns a new PyList */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; PyObject *val = NULL; PyObject *rval = PyList_New(0); Py_ssize_t next_idx; if (rval == NULL) return NULL; + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; + /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { while (idx <= end_idx) { /* read any JSON term */ @@ -872,26 +887,26 @@ idx = next_idx; /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* bail if the array is closed or we didn't get the , delimiter */ if (idx > end_idx) break; - if (str[idx] == ']') { + if (PyUnicode_READ(kind, str, idx) == ']') { break; } - else if (str[idx] != ',') { + else if (PyUnicode_READ(kind, str, idx) != ',') { raise_errmsg("Expecting , delimiter", pystr, idx); goto bail; } idx++; /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; } } - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { + /* verify that idx < end_idx, PyUnicode_READ(kind, str, idx) should be ']' */ + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { raise_errmsg("Expecting object", pystr, end_idx); goto bail; } @@ -940,16 +955,24 @@ PyInt, PyLong, or PyFloat. May return other types if parse_int or parse_float are set */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; Py_ssize_t idx = start; int is_float = 0; PyObject *rval; PyObject *numstr = NULL; PyObject *custom_func; + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; + /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { + if (PyUnicode_READ(kind, str, idx) == '-') { idx++; if (idx > end_idx) { PyErr_SetNone(PyExc_StopIteration); @@ -958,12 +981,12 @@ } /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { + if (PyUnicode_READ(kind, str, idx) >= '1' && PyUnicode_READ(kind, str, idx) <= '9') { idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; } /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { + else if (PyUnicode_READ(kind, str, idx) == '0') { idx++; } /* no integer digits, error */ @@ -973,25 +996,25 @@ } /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + if (idx < end_idx && PyUnicode_READ(kind, str, idx) == '.' && PyUnicode_READ(kind, str, idx + 1) >= '0' && PyUnicode_READ(kind, str, idx + 1) <= '9') { is_float = 1; idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; } /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + if (idx < end_idx && (PyUnicode_READ(kind, str, idx) == 'e' || PyUnicode_READ(kind, str, idx) == 'E')) { Py_ssize_t e_start = idx; idx++; /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + if (idx < end_idx && (PyUnicode_READ(kind, str, idx) == '-' || PyUnicode_READ(kind, str, idx) == '+')) idx++; /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + if (PyUnicode_READ(kind, str, idx - 1) >= '0' && PyUnicode_READ(kind, str, idx - 1) <= '9') { is_float = 1; } else { @@ -1008,7 +1031,9 @@ if (custom_func) { /* copy the section we determined to be a number */ - numstr = PyUnicode_FromUnicode(&str[start], idx - start); + numstr = PyUnicode_FromKindAndData(kind, + (char*)str + PyUnicode_KIND_SIZE(kind, start), + idx - start); if (numstr == NULL) return NULL; rval = PyObject_CallFunctionObjArgs(custom_func, numstr, NULL); @@ -1024,7 +1049,7 @@ return NULL; buf = PyBytes_AS_STRING(numstr); for (i = 0; i < n; i++) { - buf[i] = (char) str[i + start]; + buf[i] = (char) PyUnicode_READ(kind, str, i + start); } if (is_float) rval = PyFloat_FromString(numstr); @@ -1047,13 +1072,23 @@ Returns a new PyObject representation of the term. */ PyObject *res; - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t length = PyUnicode_GET_SIZE(pystr); + void *str; + int kind; + Py_ssize_t length; + + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + length = PyUnicode_GET_LENGTH(pystr); + if (idx >= length) { PyErr_SetNone(PyExc_StopIteration); return NULL; } - switch (str[idx]) { + + switch (PyUnicode_READ(kind, str, idx)) { case '"': /* string */ return scanstring_unicode(pystr, idx + 1, @@ -1077,7 +1112,7 @@ return res; case 'n': /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + if ((idx + 3 < length) && PyUnicode_READ(kind, str, idx + 1) == 'u' && PyUnicode_READ(kind, str, idx + 2) == 'l' && PyUnicode_READ(kind, str, idx + 3) == 'l') { Py_INCREF(Py_None); *next_idx_ptr = idx + 4; return Py_None; @@ -1085,7 +1120,7 @@ break; case 't': /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + if ((idx + 3 < length) && PyUnicode_READ(kind, str, idx + 1) == 'r' && PyUnicode_READ(kind, str, idx + 2) == 'u' && PyUnicode_READ(kind, str, idx + 3) == 'e') { Py_INCREF(Py_True); *next_idx_ptr = idx + 4; return Py_True; @@ -1093,7 +1128,10 @@ break; case 'f': /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + if ((idx + 4 < length) && PyUnicode_READ(kind, str, idx + 1) == 'a' && + PyUnicode_READ(kind, str, idx + 2) == 'l' && + PyUnicode_READ(kind, str, idx + 3) == 's' && + PyUnicode_READ(kind, str, idx + 4) == 'e') { Py_INCREF(Py_False); *next_idx_ptr = idx + 5; return Py_False; @@ -1101,19 +1139,33 @@ break; case 'N': /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + if ((idx + 2 < length) && PyUnicode_READ(kind, str, idx + 1) == 'a' && + PyUnicode_READ(kind, str, idx + 2) == 'N') { return _parse_constant(s, "NaN", idx, next_idx_ptr); } break; case 'I': /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + if ((idx + 7 < length) && PyUnicode_READ(kind, str, idx + 1) == 'n' && + PyUnicode_READ(kind, str, idx + 2) == 'f' && + PyUnicode_READ(kind, str, idx + 3) == 'i' && + PyUnicode_READ(kind, str, idx + 4) == 'n' && + PyUnicode_READ(kind, str, idx + 5) == 'i' && + PyUnicode_READ(kind, str, idx + 6) == 't' && + PyUnicode_READ(kind, str, idx + 7) == 'y') { return _parse_constant(s, "Infinity", idx, next_idx_ptr); } break; case '-': /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + if ((idx + 8 < length) && PyUnicode_READ(kind, str, idx + 1) == 'I' && + PyUnicode_READ(kind, str, idx + 2) == 'n' && + PyUnicode_READ(kind, str, idx + 3) == 'f' && + PyUnicode_READ(kind, str, idx + 4) == 'i' && + PyUnicode_READ(kind, str, idx + 5) == 'n' && + PyUnicode_READ(kind, str, idx + 6) == 'i' && + PyUnicode_READ(kind, str, idx + 7) == 't' && + PyUnicode_READ(kind, str, idx + 8) == 'y') { return _parse_constant(s, "-Infinity", idx, next_idx_ptr); } break; diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1867,9 +1867,7 @@ if (self->bin) { char pdata[5]; - encoded = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj), - "surrogatepass"); + encoded = PyUnicode_AsEncodedString(obj, "utf-8", "surrogatepass"); if (encoded == NULL) goto error; diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1436,10 +1436,11 @@ PyObject* uppercase_name = 0; PyObject* name; PyObject* retval; - Py_UNICODE* chk; Py_ssize_t i, len; char *uppercase_name_str; int rc; + unsigned int kind; + void *data; if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { goto finally; @@ -1454,12 +1455,16 @@ goto finally; } - len = PyUnicode_GET_SIZE(uppercase_name); - chk = PyUnicode_AS_UNICODE(uppercase_name); - for (i=0; i<len; i++, chk++) { - if ((*chk >= '0' && *chk <= '9') - || (*chk >= 'A' && *chk <= 'Z') - || (*chk == '_')) + if (PyUnicode_READY(uppercase_name)) + goto finally; + len = PyUnicode_GET_LENGTH(uppercase_name); + kind = PyUnicode_KIND(uppercase_name); + data = PyUnicode_DATA(uppercase_name); + for (i=0; i<len; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if ((ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'Z') + || (ch == '_')) { continue; } else { diff --git a/Modules/_sre.c b/Modules/_sre.c --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -163,8 +163,6 @@ /* unicode-specific character predicates */ -#if defined(HAVE_UNICODE) - #define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch)) #define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) #define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch)) @@ -176,8 +174,6 @@ return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch)); } -#endif - LOCAL(int) sre_category(SRE_CODE category, unsigned int ch) { @@ -205,7 +201,6 @@ case SRE_CATEGORY_LOC_NOT_WORD: return !SRE_LOC_IS_WORD(ch); -#if defined(HAVE_UNICODE) case SRE_CATEGORY_UNI_DIGIT: return SRE_UNI_IS_DIGIT(ch); case SRE_CATEGORY_UNI_NOT_DIGIT: @@ -222,24 +217,6 @@ return SRE_UNI_IS_LINEBREAK(ch); case SRE_CATEGORY_UNI_NOT_LINEBREAK: return !SRE_UNI_IS_LINEBREAK(ch); -#else - case SRE_CATEGORY_UNI_DIGIT: - return SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_NOT_DIGIT: - return !SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_SPACE: - return SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_NOT_SPACE: - return !SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_WORD: - return SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_NOT_WORD: - return !SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_LINEBREAK: - return SRE_IS_LINEBREAK(ch); - case SRE_CATEGORY_UNI_NOT_LINEBREAK: - return !SRE_IS_LINEBREAK(ch); -#endif } return 0; } @@ -280,6 +257,7 @@ /* generate 8-bit version */ #define SRE_CHAR unsigned char +#define SRE_CHARGET(state, buf, index) ((unsigned char*)buf)[index] #define SRE_AT sre_at #define SRE_COUNT sre_count #define SRE_CHARSET sre_charset @@ -287,15 +265,11 @@ #define SRE_MATCH sre_match #define SRE_MATCH_CONTEXT sre_match_context #define SRE_SEARCH sre_search -#define SRE_LITERAL_TEMPLATE sre_literal_template - -#if defined(HAVE_UNICODE) #define SRE_RECURSIVE #include "_sre.c" #undef SRE_RECURSIVE -#undef SRE_LITERAL_TEMPLATE #undef SRE_SEARCH #undef SRE_MATCH #undef SRE_MATCH_CONTEXT @@ -304,10 +278,15 @@ #undef SRE_COUNT #undef SRE_AT #undef SRE_CHAR - -/* generate 16-bit unicode version */ - -#define SRE_CHAR Py_UNICODE +#undef SRE_CHARGET + +/* generate 8/16/32-bit unicode version */ + +#define SRE_CHAR void +#define SRE_CHARGET(state, buf, index) \ + ((state->charsize==1) ? ((Py_UCS1*)buf)[index] : \ + (state->charsize==2) ? ((Py_UCS2*)buf)[index] : \ + ((Py_UCS4*)buf)[index]) #define SRE_AT sre_uat #define SRE_COUNT sre_ucount #define SRE_CHARSET sre_ucharset @@ -315,8 +294,6 @@ #define SRE_MATCH sre_umatch #define SRE_MATCH_CONTEXT sre_umatch_context #define SRE_SEARCH sre_usearch -#define SRE_LITERAL_TEMPLATE sre_uliteral_template -#endif #endif /* SRE_RECURSIVE */ @@ -327,7 +304,7 @@ settings */ LOCAL(int) -SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) +SRE_AT(SRE_STATE* state, char* ptr, SRE_CODE at) { /* check if pointer is at given position */ @@ -341,16 +318,16 @@ case SRE_AT_BEGINNING_LINE: return ((void*) ptr == state->beginning || - SRE_IS_LINEBREAK((int) ptr[-1])); + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, -1))); case SRE_AT_END: - return (((void*) (ptr+1) == state->end && - SRE_IS_LINEBREAK((int) ptr[0])) || + return (((void*) (ptr+state->charsize) == state->end && + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))) || ((void*) ptr == state->end)); case SRE_AT_END_LINE: return ((void*) ptr == state->end || - SRE_IS_LINEBREAK((int) ptr[0])); + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))); case SRE_AT_END_STRING: return ((void*) ptr == state->end); @@ -359,57 +336,55 @@ if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; case SRE_AT_LOC_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_LOC_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; -#if defined(HAVE_UNICODE) case SRE_AT_UNI_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_UNI_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; -#endif } @@ -476,7 +451,7 @@ count = *(set++); if (sizeof(SRE_CODE) == 2) { - block = ((unsigned char*)set)[ch >> 8]; + block = ((char*)set)[ch >> 8]; set += 128; if (set[block*16 + ((ch & 255)>>4)] & (1 << (ch & 15))) return ok; @@ -486,7 +461,7 @@ /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids * warnings when c's type supports only numbers < N+1 */ if (!(ch & ~65535)) - block = ((unsigned char*)set)[ch >> 8]; + block = ((char*)set)[ch >> 8]; else block = -1; set += 64; @@ -512,28 +487,29 @@ SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) { SRE_CODE chr; - SRE_CHAR* ptr = (SRE_CHAR *)state->ptr; - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* ptr = (char *)state->ptr; + char* end = (char *)state->end; Py_ssize_t i; /* adjust end */ if (maxcount < end - ptr && maxcount != 65535) - end = ptr + maxcount; + end = ptr + maxcount*state->charsize; switch (pattern[0]) { case SRE_OP_IN: /* repeated set */ TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); - while (ptr < end && SRE_CHARSET(pattern + 2, *ptr)) - ptr++; + while (ptr < end && + SRE_CHARSET(pattern + 2, SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; break; case SRE_OP_ANY: /* repeated dot wildcard. */ TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr)); - while (ptr < end && !SRE_IS_LINEBREAK(*ptr)) - ptr++; + while (ptr < end && !SRE_IS_LINEBREAK(SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; break; case SRE_OP_ANY_ALL: @@ -547,38 +523,38 @@ /* repeated literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr == chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) == chr) + ptr += state->charsize; break; case SRE_OP_LITERAL_IGNORE: /* repeated literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr) - ptr++; + while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) == chr) + ptr += state->charsize; break; case SRE_OP_NOT_LITERAL: /* repeated non-literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr != chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr) + ptr += state->charsize; break; case SRE_OP_NOT_LITERAL_IGNORE: /* repeated non-literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr) - ptr++; + while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) != chr) + ptr += state->charsize; break; default: /* repeated single character pattern */ TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); - while ((SRE_CHAR*) state->ptr < end) { + while ((char*) state->ptr < end) { i = SRE_MATCH(state, pattern); if (i < 0) return i; @@ -586,12 +562,12 @@ break; } TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, - (SRE_CHAR*) state->ptr - ptr)); - return (SRE_CHAR*) state->ptr - ptr; + ((char*)state->ptr - ptr)/state->charsize)); + return ((char*)state->ptr - ptr)/state->charsize; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); - return ptr - (SRE_CHAR*) state->ptr; + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, (ptr - (char*) state->ptr)/state->charsize)); + return (ptr - (char*) state->ptr)/state->charsize; } #if 0 /* not used in this release */ @@ -602,8 +578,8 @@ returns the number of SRE_CODE objects to skip if successful, 0 if no match */ - SRE_CHAR* end = state->end; - SRE_CHAR* ptr = state->ptr; + char* end = state->end; + char* ptr = state->ptr; Py_ssize_t i; /* check minimal length */ @@ -614,7 +590,7 @@ if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) { /* <length> <skip> <prefix data> <overlap data> */ for (i = 0; i < pattern[5]; i++) - if ((SRE_CODE) ptr[i] != pattern[7 + i]) + if ((SRE_CODE) SRE_CHARGET(state, ptr, i) != pattern[7 + i]) return 0; return pattern[0] + 2 * pattern[6]; } @@ -783,7 +759,7 @@ typedef struct { Py_ssize_t last_ctx_pos; Py_ssize_t jump; - SRE_CHAR* ptr; + char* ptr; SRE_CODE* pattern; Py_ssize_t count; Py_ssize_t lastmark; @@ -799,7 +775,7 @@ LOCAL(Py_ssize_t) SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) { - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* end = (char*)state->end; Py_ssize_t alloc_pos, ctx_pos = -1; Py_ssize_t i, ret = 0; Py_ssize_t jump; @@ -818,12 +794,12 @@ entrance: - ctx->ptr = (SRE_CHAR *)state->ptr; + ctx->ptr = (char *)state->ptr; if (ctx->pattern[0] == SRE_OP_INFO) { /* optimization info block */ /* <INFO> <1=skip> <2=flags> <3=min> ... */ - if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { + if (ctx->pattern[3] && (end - ctx->ptr)/state->charsize < ctx->pattern[3]) { TRACE(("reject (got %d chars, need %d)\n", (end - ctx->ptr), ctx->pattern[3])); RETURN_FAILURE; @@ -865,10 +841,10 @@ /* <LITERAL> <code> */ TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0]) + if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[0]) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_NOT_LITERAL: @@ -876,10 +852,10 @@ /* <NOT_LITERAL> <code> */ TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0]) + if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) == ctx->pattern[0]) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_SUCCESS: @@ -902,19 +878,19 @@ /* <CATEGORY> <code> */ TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0])) + if (ctx->ptr >= end || !sre_category(ctx->pattern[0], SRE_CHARGET(state, ctx->ptr, 0))) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_ANY: /* match anything (except a newline) */ /* <ANY> */ TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0])) - RETURN_FAILURE; - ctx->ptr++; + if (ctx->ptr >= end || SRE_IS_LINEBREAK(SRE_CHARGET(state, ctx->ptr, 0))) + RETURN_FAILURE; + ctx->ptr += state->charsize; break; case SRE_OP_ANY_ALL: @@ -923,47 +899,47 @@ TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr)); if (ctx->ptr >= end) RETURN_FAILURE; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_IN: /* match set member (or non_member) */ /* <IN> <skip> <set> */ TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr)) - RETURN_FAILURE; + if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, SRE_CHARGET(state, ctx->ptr, 0))) + RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_LITERAL_IGNORE: TRACE(("|%p|%p|LITERAL_IGNORE %d\n", ctx->pattern, ctx->ptr, ctx->pattern[0])); if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*ctx->pattern)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*ctx->pattern)) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_NOT_LITERAL_IGNORE: TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); if (ctx->ptr >= end || - state->lower(*ctx->ptr) == state->lower(*ctx->pattern)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) == state->lower(*ctx->pattern)) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_IN_IGNORE: TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern+1, - (SRE_CODE)state->lower(*ctx->ptr))) + (SRE_CODE)state->lower(SRE_CHARGET(state, ctx->ptr, 0)))) RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_JUMP: @@ -986,11 +962,11 @@ for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { if (ctx->pattern[1] == SRE_OP_LITERAL && (ctx->ptr >= end || - (SRE_CODE) *ctx->ptr != ctx->pattern[2])) + (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[2])) continue; if (ctx->pattern[1] == SRE_OP_IN && (ctx->ptr >= end || - !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) + !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0)))) continue; state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); @@ -1021,7 +997,7 @@ TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->ptr + state->charsize * ctx->pattern[1] > end) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1030,7 +1006,7 @@ RETURN_ON_ERROR(ret); DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); ctx->count = ret; - ctx->ptr += ctx->count; + ctx->ptr += state->charsize * ctx->count; /* when we arrive here, count contains the number of matches, and ctx->ptr points to the tail of the target @@ -1054,8 +1030,9 @@ ctx->u.chr = ctx->pattern[ctx->pattern[0]+1]; for (;;) { while (ctx->count >= (Py_ssize_t) ctx->pattern[1] && - (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) { - ctx->ptr--; + (ctx->ptr >= end || + SRE_CHARGET(state, ctx->ptr, 0) != ctx->u.chr)) { + ctx->ptr -= state->charsize; ctx->count--; } if (ctx->count < (Py_ssize_t) ctx->pattern[1]) @@ -1070,7 +1047,7 @@ LASTMARK_RESTORE(); - ctx->ptr--; + ctx->ptr -= state->charsize; ctx->count--; } @@ -1084,7 +1061,7 @@ RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - ctx->ptr--; + ctx->ptr -= state->charsize; ctx->count--; LASTMARK_RESTORE(); } @@ -1104,7 +1081,7 @@ TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->ptr + state->charsize * ctx->pattern[1] > end) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1121,7 +1098,7 @@ RETURN_FAILURE; /* advance past minimum matches of repeat */ ctx->count = ret; - ctx->ptr += ctx->count; + ctx->ptr += state->charsize * ctx->count; } if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { @@ -1148,7 +1125,7 @@ if (ret == 0) break; assert(ret == 1); - ctx->ptr++; + ctx->ptr += state->charsize; ctx->count++; LASTMARK_RESTORE(); } @@ -1320,14 +1297,16 @@ if (groupref >= state->lastmark) { RETURN_FAILURE; } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + char* p = (char*) state->mark[groupref]; + char* e = (char*) state->mark[groupref+1]; if (!p || !e || e < p) RETURN_FAILURE; while (p < e) { - if (ctx->ptr >= end || *ctx->ptr != *p) + if (ctx->ptr >= end || + SRE_CHARGET(state, ctx->ptr, 0) != SRE_CHARGET(state, p, 0)) RETURN_FAILURE; - p++; ctx->ptr++; + p += state->charsize; + ctx->ptr += state->charsize; } } } @@ -1344,15 +1323,16 @@ if (groupref >= state->lastmark) { RETURN_FAILURE; } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + char* p = (char*) state->mark[groupref]; + char* e = (char*) state->mark[groupref+1]; if (!p || !e || e < p) RETURN_FAILURE; while (p < e) { if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*p)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*p)) RETURN_FAILURE; - p++; ctx->ptr++; + p++; + ctx->ptr += state->charsize; } } } @@ -1386,7 +1366,7 @@ /* <ASSERT> <skip> <back> <pattern> */ TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; + state->ptr = ctx->ptr - state->charsize * ctx->pattern[1]; if (state->ptr < state->beginning) RETURN_FAILURE; DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2); @@ -1399,7 +1379,7 @@ /* <ASSERT_NOT> <skip> <back> <pattern> */ TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; + state->ptr = ctx->ptr - state->charsize * ctx->pattern[1]; if (state->ptr >= state->beginning) { DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { @@ -1481,8 +1461,8 @@ LOCAL(Py_ssize_t) SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) { - SRE_CHAR* ptr = (SRE_CHAR *)state->start; - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* ptr = (char*)state->start; + char* end = (char*)state->end; Py_ssize_t status = 0; Py_ssize_t prefix_len = 0; Py_ssize_t prefix_skip = 0; @@ -1500,9 +1480,9 @@ if (pattern[3] > 1) { /* adjust end point (but make sure we leave at least one character in there, so literal search will work) */ - end -= pattern[3]-1; + end -= (pattern[3]-1) * state->charsize; if (end <= ptr) - end = ptr+1; + end = ptr + state->charsize; } if (flags & SRE_INFO_PREFIX) { @@ -1528,10 +1508,10 @@ /* pattern starts with a known prefix. use the overlap table to skip forward as fast as we possibly can */ Py_ssize_t i = 0; - end = (SRE_CHAR *)state->end; + end = (char *)state->end; while (ptr < end) { for (;;) { - if ((SRE_CODE) ptr[0] != prefix[i]) { + if ((SRE_CODE) SRE_CHARGET(state, ptr, 0) != prefix[i]) { if (!i) break; else @@ -1540,8 +1520,8 @@ if (++i == prefix_len) { /* found a potential match */ TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr)); - state->start = ptr + 1 - prefix_len; - state->ptr = ptr + 1 - prefix_len + prefix_skip; + state->start = ptr - (prefix_len - 1) * state->charsize; + state->ptr = ptr - (prefix_len - prefix_skip - 1) * state->charsize; if (flags & SRE_INFO_LITERAL) return 1; /* we got all of it */ status = SRE_MATCH(state, pattern + 2*prefix_skip); @@ -1553,7 +1533,7 @@ break; } } - ptr++; + ptr += state->charsize; } return 0; } @@ -1563,15 +1543,16 @@ /* pattern starts with a literal character. this is used for short prefixes, and if fast search is disabled */ SRE_CODE chr = pattern[1]; - end = (SRE_CHAR *)state->end; + end = (char*)state->end; for (;;) { - while (ptr < end && (SRE_CODE) ptr[0] != chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr) + ptr += state->charsize; if (ptr >= end) return 0; TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); state->start = ptr; - state->ptr = ++ptr; + ptr += state->charsize; + state->ptr = ptr; if (flags & SRE_INFO_LITERAL) return 1; /* we got all of it */ status = SRE_MATCH(state, pattern + 2); @@ -1580,10 +1561,10 @@ } } else if (charset) { /* pattern starts with a character from a known set */ - end = (SRE_CHAR *)state->end; + end = (char*)state->end; for (;;) { - while (ptr < end && !SRE_CHARSET(charset, ptr[0])) - ptr++; + while (ptr < end && !SRE_CHARSET(charset, SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; if (ptr >= end) return 0; TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr)); @@ -1592,13 +1573,14 @@ status = SRE_MATCH(state, pattern); if (status != 0) break; - ptr++; + ptr += state->charsize; } } else /* general case */ while (ptr <= end) { TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); - state->start = state->ptr = ptr++; + state->start = state->ptr = ptr; + ptr += state->charsize; status = SRE_MATCH(state, pattern); if (status != 0) break; @@ -1607,16 +1589,6 @@ return status; } -LOCAL(int) -SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len) -{ - /* check if given string is a literal template (i.e. no escapes) */ - while (len-- > 0) - if (*ptr++ == '\\') - return 0; - return 1; -} - #if !defined(SRE_RECURSIVE) /* -------------------------------------------------------------------- */ @@ -1626,6 +1598,23 @@ static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int); static PyObject*pattern_scanner(PatternObject*, PyObject*); +static int +sre_literal_template(int charsize, char* ptr, Py_ssize_t len) +{ + /* check if given string is a literal template (i.e. no escapes) */ + struct { + int charsize; + } state = { + charsize + }; + while (len-- > 0) { + if (SRE_CHARGET((&state), ptr, 0) == '\\') + return 0; + ptr += charsize; + } + return 1; +} + static PyObject * sre_codesize(PyObject* self, PyObject *unused) { @@ -1641,11 +1630,7 @@ if (flags & SRE_FLAG_LOCALE) return Py_BuildValue("i", sre_lower_locale(character)); if (flags & SRE_FLAG_UNICODE) -#if defined(HAVE_UNICODE) return Py_BuildValue("i", sre_lower_unicode(character)); -#else - return Py_BuildValue("i", sre_lower_locale(character)); -#endif return Py_BuildValue("i", sre_lower(character)); } @@ -1664,7 +1649,8 @@ } static void* -getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) +getstring(PyObject* string, Py_ssize_t* p_length, + int* p_logical_charsize, int* p_charsize) { /* given a python object, return a data pointer, a length (in characters), and a character size. return NULL if the object @@ -1679,9 +1665,12 @@ /* Unicode objects do not support the buffer API. So, get the data directly instead. */ if (PyUnicode_Check(string)) { - ptr = (void *)PyUnicode_AS_DATA(string); - *p_length = PyUnicode_GET_SIZE(string); - *p_charsize = sizeof(Py_UNICODE); + if (PyUnicode_READY(string) == -1) + return NULL; + ptr = PyUnicode_DATA(string); + *p_length = PyUnicode_GET_LENGTH(string); + *p_charsize = PyUnicode_CHARACTER_SIZE(string); + *p_logical_charsize = 4; return ptr; } @@ -1713,10 +1702,8 @@ if (PyBytes_Check(string) || bytes == size) charsize = 1; -#if defined(HAVE_UNICODE) else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) charsize = sizeof(Py_UNICODE); -#endif else { PyErr_SetString(PyExc_TypeError, "buffer size mismatch"); return NULL; @@ -1724,6 +1711,7 @@ *p_length = size; *p_charsize = charsize; + *p_logical_charsize = charsize; if (ptr == NULL) { PyErr_SetString(PyExc_ValueError, @@ -1739,7 +1727,7 @@ /* prepare state object */ Py_ssize_t length; - int charsize; + int logical_charsize, charsize; void* ptr; memset(state, 0, sizeof(SRE_STATE)); @@ -1747,16 +1735,16 @@ state->lastmark = -1; state->lastindex = -1; - ptr = getstring(string, &length, &charsize); + ptr = getstring(string, &length, &logical_charsize, &charsize); if (!ptr) return NULL; - if (charsize == 1 && pattern->charsize > 1) { + if (logical_charsize == 1 && pattern->logical_charsize > 1) { PyErr_SetString(PyExc_TypeError, "can't use a string pattern on a bytes-like object"); return NULL; } - if (charsize > 1 && pattern->charsize == 1) { + if (logical_charsize > 1 && pattern->logical_charsize == 1) { PyErr_SetString(PyExc_TypeError, "can't use a bytes pattern on a string-like object"); return NULL; @@ -1773,6 +1761,7 @@ else if (end > length) end = length; + state->logical_charsize = logical_charsize; state->charsize = charsize; state->beginning = ptr; @@ -1788,11 +1777,7 @@ if (pattern->flags & SRE_FLAG_LOCALE) state->lower = sre_lower_locale; else if (pattern->flags & SRE_FLAG_UNICODE) -#if defined(HAVE_UNICODE) state->lower = sre_lower_unicode; -#else - state->lower = sre_lower_locale; -#endif else state->lower = sre_lower; @@ -1891,12 +1876,10 @@ TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_match(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_umatch(&state, PatternObject_GetCode(self)); -#endif } TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); @@ -1928,12 +1911,10 @@ TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr)); - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); @@ -2075,12 +2056,10 @@ state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } if (PyErr_Occurred()) @@ -2205,12 +2184,10 @@ state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } if (PyErr_Occurred()) @@ -2295,7 +2272,7 @@ int status; Py_ssize_t n; Py_ssize_t i, b, e; - int bint; + int logical_charsize, charsize; int filter_is_callable; if (PyCallable_Check(ptemplate)) { @@ -2306,16 +2283,10 @@ } else { /* if not callable, check if it's a literal string */ int literal; - ptr = getstring(ptemplate, &n, &bint); - b = bint; + ptr = getstring(ptemplate, &n, &logical_charsize, &charsize); + b = charsize; if (ptr) { - if (b == 1) { - literal = sre_literal_template((unsigned char *)ptr, n); - } else { -#if defined(HAVE_UNICODE) - literal = sre_uliteral_template((Py_UNICODE *)ptr, n); -#endif - } + literal = sre_literal_template(b, ptr, n); } else { PyErr_Clear(); literal = 0; @@ -2357,12 +2328,10 @@ state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } if (PyErr_Occurred()) @@ -2694,15 +2663,18 @@ return NULL; } - if (pattern == Py_None) - self->charsize = -1; - else { - Py_ssize_t p_length; - if (!getstring(pattern, &p_length, &self->charsize)) { - Py_DECREF(self); - return NULL; - } - } + if (pattern == Py_None) { + self->logical_charsize = -1; + self->charsize = -1; + } + else { + Py_ssize_t p_length; + if (!getstring(pattern, &p_length, &self->logical_charsize, + &self->charsize)) { + Py_DECREF(self); + return NULL; + } + } Py_INCREF(pattern); self->pattern = pattern; @@ -3746,12 +3718,10 @@ state->ptr = state->start; - if (state->charsize == 1) { + if (state->logical_charsize == 1) { status = sre_match(state, PatternObject_GetCode(self->pattern)); } else { -#if defined(HAVE_UNICODE) status = sre_umatch(state, PatternObject_GetCode(self->pattern)); -#endif } if (PyErr_Occurred()) return NULL; @@ -3779,12 +3749,10 @@ state->ptr = state->start; - if (state->charsize == 1) { + if (state->logical_charsize == 1) { status = sre_search(state, PatternObject_GetCode(self->pattern)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(state, PatternObject_GetCode(self->pattern)); -#endif } if (PyErr_Occurred()) return NULL; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1355,7 +1355,7 @@ test_Z_code(PyObject *self) { PyObject *tuple, *obj; - Py_UNICODE *value1, *value2; + const Py_UNICODE *value1, *value2; Py_ssize_t len1, len2; tuple = PyTuple_New(2); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -80,18 +80,6 @@ #error "Tk older than 8.3.1 not supported" #endif -/* Unicode conversion assumes that Tcl_UniChar is two bytes. - We cannot test this directly, so we test UTF-8 size instead, - expecting that TCL_UTF_MAX is changed if Tcl ever supports - either UTF-16 or UCS-4. - Redhat 8 sets TCL_UTF_MAX to 6, and uses wchar_t for - Tcl_Unichar. This is also ok as long as Python uses UCS-4, - as well. -*/ -#if TCL_UTF_MAX != 3 && !(defined(Py_UNICODE_WIDE) && TCL_UTF_MAX==6) -#error "unsupported Tcl configuration" -#endif - #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -975,38 +963,44 @@ return result; } else if (PyUnicode_Check(value)) { - Py_UNICODE *inbuf = PyUnicode_AS_UNICODE(value); - Py_ssize_t size = PyUnicode_GET_SIZE(value); - /* This #ifdef assumes that Tcl uses UCS-2. - See TCL_UTF_MAX test above. */ -#if defined(Py_UNICODE_WIDE) && TCL_UTF_MAX == 3 + void *inbuf; + Py_ssize_t size; + int kind; Tcl_UniChar *outbuf = NULL; Py_ssize_t i; - size_t allocsize = ((size_t)size) * sizeof(Tcl_UniChar); - if (allocsize >= size) - outbuf = (Tcl_UniChar*)ckalloc(allocsize); + size_t allocsize; + + if (PyUnicode_READY(value) == -1) + return NULL; + + inbuf = PyUnicode_DATA(value); + size = PyUnicode_GET_LENGTH(value); + kind = PyUnicode_KIND(value); + allocsize = ((size_t)size) * sizeof(Tcl_UniChar); + outbuf = (Tcl_UniChar*)ckalloc(allocsize); /* Else overflow occurred, and we take the next exit */ if (!outbuf) { PyErr_NoMemory(); return NULL; } for (i = 0; i < size; i++) { - if (inbuf[i] >= 0x10000) { + Py_UCS4 ch = PyUnicode_READ(kind, inbuf, i); + /* We cannot test for sizeof(Tcl_UniChar) directly, + so we test for UTF-8 size instead. */ +#if TCL_UTF_MAX == 3 + if (ch >= 0x10000) { /* Tcl doesn't do UTF-16, yet. */ PyErr_SetString(PyExc_ValueError, "unsupported character"); ckfree(FREECAST outbuf); return NULL; +#endif } - outbuf[i] = inbuf[i]; + outbuf[i] = ch; } result = Tcl_NewUnicodeObj(outbuf, size); ckfree(FREECAST outbuf); return result; -#else - return Tcl_NewUnicodeObj(inbuf, size); -#endif - } else if(PyTclObject_Check(value)) { Tcl_Obj *v = ((PyTclObject*)value)->value; @@ -1088,24 +1082,14 @@ } if (value->typePtr == app->StringType) { -#if defined(Py_UNICODE_WIDE) && TCL_UTF_MAX==3 - PyObject *result; - int size; - Tcl_UniChar *input; - Py_UNICODE *output; - - size = Tcl_GetCharLength(value); - result = PyUnicode_FromUnicode(NULL, size); - if (!result) - return NULL; - input = Tcl_GetUnicode(value); - output = PyUnicode_AS_UNICODE(result); - while (size--) - *output++ = *input++; - return result; +#if TCL_UTF_MAX==3 + return PyUnicode_FromKindAndData( + PyUnicode_2BYTE_KIND, Tcl_GetUnicode(value), + Tcl_GetCharLength(value)); #else - return PyUnicode_FromUnicode(Tcl_GetUnicode(value), - Tcl_GetCharLength(value)); + return PyUnicode_FromKindAndData( + PyUnicode_4BYTE_KIND, Tcl_GetUnicode(value), + Tcl_GetCharLength(value)); #endif } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2810,9 +2810,9 @@ PyInit_array(void) { PyObject *m; + char buffer[PY_ARRAY_LENGTH(descriptors)], *p; PyObject *typecodes; Py_ssize_t size = 0; - register Py_UNICODE *p; struct arraydescr *descr; if (PyType_Ready(&Arraytype) < 0) @@ -2831,13 +2831,13 @@ size++; } - typecodes = PyUnicode_FromStringAndSize(NULL, size); - p = PyUnicode_AS_UNICODE(typecodes); + p = buffer; for (descr = descriptors; descr->typecode != '\0'; descr++) { *p++ = (char)descr->typecode; } + typecodes = PyUnicode_DecodeASCII(buffer, p - buffer, NULL); - PyModule_AddObject(m, "typecodes", (PyObject *)typecodes); + PyModule_AddObject(m, "typecodes", typecodes); if (PyErr_Occurred()) { Py_DECREF(m); diff --git a/Modules/md5module.c b/Modules/md5module.c --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -376,7 +376,7 @@ unsigned char digest[MD5_DIGESTSIZE]; struct md5_state temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -384,14 +384,10 @@ md5_done(&temp, digest); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, MD5_DIGESTSIZE * 2); + retval = PyUnicode_New(MD5_DIGESTSIZE * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<MD5_DIGESTSIZE; i++) { diff --git a/Modules/operator.c b/Modules/operator.c --- a/Modules/operator.c +++ b/Modules/operator.c @@ -402,7 +402,8 @@ for (idx = 0; idx < nattrs; ++idx) { PyObject *item = PyTuple_GET_ITEM(args, idx); Py_ssize_t item_len; - Py_UNICODE *item_buffer; + void *data; + unsigned int kind; int dot_count; if (!PyUnicode_Check(item)) { @@ -411,13 +412,18 @@ Py_DECREF(attr); return NULL; } - item_len = PyUnicode_GET_SIZE(item); - item_buffer = PyUnicode_AS_UNICODE(item); + if (PyUnicode_READY(item)) { + Py_DECREF(attr); + return NULL; + } + item_len = PyUnicode_GET_LENGTH(item); + kind = PyUnicode_KIND(item); + data = PyUnicode_DATA(item); /* check whethere the string is dotted */ dot_count = 0; for (char_idx = 0; char_idx < item_len; ++char_idx) { - if (item_buffer[char_idx] == (Py_UNICODE)'.') + if (PyUnicode_READ(kind, data, char_idx) == '.') ++dot_count; } @@ -438,12 +444,12 @@ } for (; dot_count > 0; --dot_count) { - while (item_buffer[unibuff_till] != (Py_UNICODE)'.') { + while (PyUnicode_READ(kind, data, unibuff_till) != '.') { ++unibuff_till; } - attr_chain_item = PyUnicode_FromUnicode( - item_buffer + unibuff_from, - unibuff_till - unibuff_from); + attr_chain_item = PyUnicode_Substring(item, + unibuff_from, + unibuff_till); if (attr_chain_item == NULL) { Py_DECREF(attr_chain); Py_DECREF(attr); @@ -456,9 +462,8 @@ } /* now add the last dotless name */ - attr_chain_item = PyUnicode_FromUnicode( - item_buffer + unibuff_from, - item_len - unibuff_from); + attr_chain_item = PyUnicode_Substring(item, + unibuff_from, item_len); if (attr_chain_item == NULL) { Py_DECREF(attr_chain); Py_DECREF(attr); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1102,17 +1102,22 @@ PyUnicodeObject *_u_string = NULL; int result = 0; int i; + int kind; + void *data; /* Yes, supports only 8bit encodings */ _u_string = (PyUnicodeObject *) PyUnicode_Decode(template_buffer, 256, name, "replace"); - if (_u_string == NULL) + if (_u_string == NULL || PyUnicode_READY(_u_string) == -1) return result; + kind = PyUnicode_KIND(_u_string); + data = PyUnicode_DATA(_u_string); + for (i = 0; i < 256; i++) { /* Stupid to access directly, but fast */ - Py_UNICODE c = _u_string->str[i]; + Py_UCS4 c = PyUnicode_READ(kind, data, i); if (c == Py_UNICODE_REPLACEMENT_CHARACTER) info->map[i] = -1; else @@ -1229,7 +1234,7 @@ static PyObject * xmlparse_getattro(xmlparseobject *self, PyObject *nameobj) { - Py_UNICODE *name; + const Py_UNICODE *name; int handlernum = -1; if (!PyUnicode_Check(nameobj)) diff --git a/Modules/sha1module.c b/Modules/sha1module.c --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -352,7 +352,7 @@ unsigned char digest[SHA1_DIGESTSIZE]; struct sha1_state temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -360,14 +360,10 @@ sha1_done(&temp, digest); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, SHA1_DIGESTSIZE * 2); + retval = PyUnicode_New(SHA1_DIGESTSIZE * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<SHA1_DIGESTSIZE; i++) { diff --git a/Modules/sha256module.c b/Modules/sha256module.c --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -445,7 +445,7 @@ unsigned char digest[SHA_DIGESTSIZE]; SHAobject temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -453,14 +453,10 @@ sha_final(digest, &temp); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, self->digestsize * 2); + retval = PyUnicode_New(self->digestsize * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<self->digestsize; i++) { diff --git a/Modules/sha512module.c b/Modules/sha512module.c --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -511,7 +511,7 @@ unsigned char digest[SHA_DIGESTSIZE]; SHAobject temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -519,14 +519,10 @@ sha512_final(digest, &temp); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, self->digestsize * 2); + retval = PyUnicode_New(self->digestsize * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for (i=j=0; i<self->digestsize; i++) { diff --git a/Modules/sre.h b/Modules/sre.h --- a/Modules/sre.h +++ b/Modules/sre.h @@ -30,7 +30,8 @@ PyObject* pattern; /* pattern source (or None) */ int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ - int charsize; /* pattern charsize (or -1) */ + int logical_charsize; /* pattern charsize (or -1) */ + int charsize; /* pattern code */ Py_ssize_t codesize; SRE_CODE code[1]; @@ -71,6 +72,7 @@ PyObject* string; Py_ssize_t pos, endpos; /* character size */ + int logical_charsize; /* kind of thing: 1 - bytes, 2/4 - unicode */ int charsize; /* registers */ Py_ssize_t lastindex; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -70,7 +70,7 @@ Py_ssize_t argv_len, scriptlen; PyObject *scriptobj; - Py_UNICODE *atslash, *atstart; + Py_ssize_t slash; PyObject *argv = PySys_GetObject("argv"); if (argv == NULL) { @@ -95,11 +95,13 @@ return(NULL); } - atstart = PyUnicode_AS_UNICODE(scriptobj); - atslash = Py_UNICODE_strrchr(atstart, SEP); - if (atslash) { - return(PyUnicode_FromUnicode(atslash + 1, - scriptlen - (atslash - atstart) - 1)); + slash = PyUnicode_FindChar(scriptobj, SEP, + 0, PyUnicode_GET_LENGTH(scriptobj), -1); + if (slash == -2) + return NULL; + if (slash != -1) { + return PyUnicode_Substring(scriptobj, slash, + PyUnicode_GET_LENGTH(scriptobj)); } else { Py_INCREF(scriptobj); return(scriptobj); diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -92,16 +92,13 @@ static Py_UCS4 getuchar(PyUnicodeObject *obj) { - Py_UNICODE *v = PyUnicode_AS_UNICODE(obj); - - if (PyUnicode_GET_SIZE(obj) == 1) - return *v; -#ifndef Py_UNICODE_WIDE - else if ((PyUnicode_GET_SIZE(obj) == 2) && - (0xD800 <= v[0] && v[0] <= 0xDBFF) && - (0xDC00 <= v[1] && v[1] <= 0xDFFF)) - return (((v[0] & 0x3FF)<<10) | (v[1] & 0x3FF)) + 0x10000; -#endif + if (PyUnicode_READY(obj)) + return (Py_UCS4)-1; + if (PyUnicode_GET_LENGTH(obj) == 1) { + if (PyUnicode_READY(obj)) + return (Py_UCS4)-1; + return PyUnicode_READ_CHAR(obj, 0); + } PyErr_SetString(PyExc_TypeError, "need a single Unicode character as parameter"); return (Py_UCS4)-1; @@ -1142,7 +1139,6 @@ unicodedata_lookup(PyObject* self, PyObject* args) { Py_UCS4 code; - Py_UNICODE str[2]; char* name; int namelen; @@ -1155,15 +1151,7 @@ return NULL; } -#ifndef Py_UNICODE_WIDE - if (code >= 0x10000) { - str[0] = 0xd800 + ((code - 0x10000) >> 10); - str[1] = 0xdc00 + ((code - 0x10000) & 0x3ff); - return PyUnicode_FromUnicode(str, 2); - } -#endif - str[0] = (Py_UNICODE) code; - return PyUnicode_FromUnicode(str, 1); + return PyUnicode_FromOrdinal(code); } /* XXX Add doc strings. */ diff --git a/Modules/zipimport.c b/Modules/zipimport.c --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -64,7 +64,7 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) { PyObject *pathobj, *files; - Py_UNICODE *path, *p, *prefix, buf[MAXPATHLEN+2]; + Py_UCS4 *path, *p, *prefix, buf[MAXPATHLEN+2]; Py_ssize_t len; if (!_PyArg_NoKeywords("zipimporter()", kwds)) @@ -74,8 +74,11 @@ PyUnicode_FSDecoder, &pathobj)) return -1; + if (PyUnicode_READY(pathobj) == -1) + return -1; + /* copy path to buf */ - len = PyUnicode_GET_SIZE(pathobj); + len = PyUnicode_GET_LENGTH(pathobj); if (len == 0) { PyErr_SetString(ZipImportError, "archive path is empty"); goto error; @@ -85,7 +88,8 @@ "archive path too long"); goto error; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(pathobj)); + if (!PyUnicode_AsUCS4(pathobj, buf, PY_ARRAY_LENGTH(buf), 1)) + goto error; #ifdef ALTSEP for (p = buf; *p; p++) { @@ -101,7 +105,8 @@ int rv; if (pathobj == NULL) { - pathobj = PyUnicode_FromUnicode(buf, len); + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (pathobj == NULL) goto error; } @@ -116,7 +121,7 @@ else if (PyErr_Occurred()) goto error; /* back up one path element */ - p = Py_UNICODE_strrchr(buf, SEP); + p = Py_UCS4_strrchr(buf, SEP); if (prefix != NULL) *prefix = SEP; if (p == NULL) @@ -148,7 +153,7 @@ if (prefix != NULL) { prefix++; - len = Py_UNICODE_strlen(prefix); + len = Py_UCS4_strlen(prefix); if (prefix[len-1] != SEP) { /* add trailing SEP */ prefix[len] = SEP; @@ -158,7 +163,8 @@ } else len = 0; - self->prefix = PyUnicode_FromUnicode(prefix, len); + self->prefix = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + prefix, len); if (self->prefix == NULL) goto error; @@ -193,7 +199,7 @@ { if (self->archive == NULL) return PyUnicode_FromString("<zipimporter object \"???\">"); - else if (self->prefix != NULL && PyUnicode_GET_SIZE(self->prefix) != 0) + else if (self->prefix != NULL && PyUnicode_GET_LENGTH(self->prefix) != 0) return PyUnicode_FromFormat("<zipimporter object \"%U%c%U\">", self->archive, SEP, self->prefix); else @@ -206,16 +212,24 @@ get_subname(PyObject *fullname) { Py_ssize_t len; - Py_UNICODE *subname; - subname = Py_UNICODE_strrchr(PyUnicode_AS_UNICODE(fullname), '.'); + Py_UCS4 *subname, *fullname_ucs4; + fullname_ucs4 = PyUnicode_AsUCS4Copy(fullname); + if (!fullname_ucs4) + return NULL; + subname = Py_UCS4_strrchr(fullname_ucs4, '.'); if (subname == NULL) { + PyMem_Free(fullname_ucs4); Py_INCREF(fullname); return fullname; } else { + PyObject *result; subname++; - len = PyUnicode_GET_SIZE(fullname); - len -= subname - PyUnicode_AS_UNICODE(fullname); - return PyUnicode_FromUnicode(subname, len); + len = PyUnicode_GET_LENGTH(fullname); + len -= subname - fullname_ucs4; + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); + PyMem_Free(fullname_ucs4); + return result; } } @@ -228,23 +242,29 @@ make_filename(PyObject *prefix, PyObject *name) { PyObject *pathobj; - Py_UNICODE *p; + Py_UCS4 *p, *buf; + Py_ssize_t len; - pathobj = PyUnicode_FromUnicode(NULL, - PyUnicode_GET_SIZE(prefix) - + PyUnicode_GET_SIZE(name)); - if (pathobj == NULL) + len = PyUnicode_GET_LENGTH(prefix) + PyUnicode_GET_LENGTH(name) + 1; + p = buf = PyMem_Malloc(sizeof(Py_UCS4) * len); + if (buf == NULL) { + PyErr_NoMemory(); return NULL; + } - p = PyUnicode_AS_UNICODE(pathobj); - - Py_UNICODE_strcpy(p, PyUnicode_AS_UNICODE(prefix)); - p += PyUnicode_GET_SIZE(prefix); - Py_UNICODE_strcpy(p, PyUnicode_AS_UNICODE(name)); + if (!PyUnicode_AsUCS4(prefix, p, len, 0)) + return NULL; + p += PyUnicode_GET_LENGTH(prefix); + len -= PyUnicode_GET_LENGTH(prefix); + if (!PyUnicode_AsUCS4(name, p, len, 1)) + return NULL; for (; *p; p++) { if (*p == '.') *p = SEP; } + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, p-buf); + PyMem_Free(buf); return pathobj; } @@ -330,6 +350,8 @@ if (!PyArg_ParseTuple(args, "U:zipimporter.load_module", &fullname)) return NULL; + if (PyUnicode_READY(fullname) == -1) + return NULL; code = get_module_code(self, fullname, &ispackage, &modpath); if (code == NULL) @@ -426,46 +448,53 @@ return PyBool_FromLong(mi == MI_PACKAGE); } + static PyObject * zipimporter_get_data(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *pathobj, *key; - const Py_UNICODE *path; + const Py_UCS4 *path; #ifdef ALTSEP - Py_UNICODE *p, buf[MAXPATHLEN + 1]; + Py_UCS4 *p; #endif - Py_UNICODE *archive; PyObject *toc_entry; Py_ssize_t path_len, len; + Py_UCS4 buf[MAXPATHLEN + 1], archive[MAXPATHLEN + 1]; if (!PyArg_ParseTuple(args, "U:zipimporter.get_data", &pathobj)) return NULL; - path_len = PyUnicode_GET_SIZE(pathobj); - path = PyUnicode_AS_UNICODE(pathobj); -#ifdef ALTSEP + if (PyUnicode_READY(pathobj) == -1) + return NULL; + + path_len = PyUnicode_GET_LENGTH(pathobj); if (path_len >= MAXPATHLEN) { PyErr_SetString(ZipImportError, "path too long"); return NULL; } - Py_UNICODE_strcpy(buf, path); + if (!PyUnicode_AsUCS4(pathobj, buf, PY_ARRAY_LENGTH(buf), 1)) + return NULL; + path = buf; +#ifdef ALTSEP for (p = buf; *p; p++) { if (*p == ALTSEP) *p = SEP; } - path = buf; #endif - archive = PyUnicode_AS_UNICODE(self->archive); - len = PyUnicode_GET_SIZE(self->archive); - if ((size_t)len < Py_UNICODE_strlen(path) && - Py_UNICODE_strncmp(path, archive, len) == 0 && - path[len] == SEP) { - path += len + 1; - path_len -= len + 1; + len = PyUnicode_GET_LENGTH(self->archive); + if ((size_t)len < Py_UCS4_strlen(path)) { + if (!PyUnicode_AsUCS4(self->archive, archive, PY_ARRAY_LENGTH(archive), 1)) + return NULL; + if (Py_UCS4_strncmp(path, archive, len) == 0 && + path[len] == SEP) { + path += len + 1; + path_len -= len + 1; + } } - key = PyUnicode_FromUnicode(path, path_len); + key = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + path, path_len); if (key == NULL) return NULL; toc_entry = PyDict_GetItem(self->files, key); @@ -725,9 +754,10 @@ unsigned short flags; long compress, crc, data_size, file_size, file_offset, date, time; long header_offset, name_size, header_size, header_position; - long i, l, count; + long l, count; + Py_ssize_t i; size_t length; - Py_UNICODE path[MAXPATHLEN + 5]; + Py_UCS4 path[MAXPATHLEN + 5]; char name[MAXPATHLEN + 5]; PyObject *nameobj = NULL; char *p, endof_central_dir[22]; @@ -736,12 +766,13 @@ const char *charset; int bootstrap; - if (PyUnicode_GET_SIZE(archive) > MAXPATHLEN) { + if (PyUnicode_GET_LENGTH(archive) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "Zip path name is too long"); return NULL; } - Py_UNICODE_strcpy(path, PyUnicode_AS_UNICODE(archive)); + if (!PyUnicode_AsUCS4(archive, path, PY_ARRAY_LENGTH(path), 1)) + return NULL; fp = _Py_fopen(archive, "rb"); if (fp == NULL) { @@ -771,7 +802,7 @@ if (files == NULL) goto error; - length = Py_UNICODE_strlen(path); + length = Py_UCS4_strlen(path); path[length] = SEP; /* Start of Central Directory */ @@ -802,7 +833,7 @@ name_size = MAXPATHLEN; p = name; - for (i = 0; i < name_size; i++) { + for (i = 0; i < (Py_ssize_t)name_size; i++) { *p = (char)getc(fp); if (*p == '/') *p = SEP; @@ -827,6 +858,8 @@ else charset = "cp437"; nameobj = PyUnicode_Decode(name, name_size, charset, NULL); + if (PyUnicode_READY(nameobj) == -1) + goto error; if (nameobj == NULL) { if (bootstrap) PyErr_Format(PyExc_NotImplementedError, @@ -835,11 +868,12 @@ PY_MAJOR_VERSION, PY_MINOR_VERSION); goto error; } - Py_UNICODE_strncpy(path + length + 1, - PyUnicode_AS_UNICODE(nameobj), - MAXPATHLEN - length - 1); - - pathobj = PyUnicode_FromUnicode(path, Py_UNICODE_strlen(path)); + for (i = 0; (i < MAXPATHLEN - length - 1) && + (i < PyUnicode_GET_LENGTH(nameobj)); i++) + path[length + 1 + i] = PyUnicode_READ_CHAR(nameobj, i); + path[length + 1 + i] = 0; + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + path, Py_UCS4_strlen(path)); if (pathobj == NULL) goto error; t = Py_BuildValue("Niiiiiii", pathobj, compress, data_size, @@ -1148,8 +1182,11 @@ time_t mtime; /* strip 'c' or 'o' from *.py[co] */ - stripped = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(path), - PyUnicode_GET_SIZE(path) - 1); + if (PyUnicode_READY(path) == -1) + return (time_t)-1; + stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), + PyUnicode_DATA(path), + PyUnicode_GET_LENGTH(path) - 1); if (stripped == NULL) return (time_t)-1; diff --git a/Objects/abstract.c b/Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1379,9 +1379,7 @@ PyBytes_GET_SIZE(o)); if (PyUnicode_Check(o)) /* The above check is done in PyLong_FromUnicode(). */ - return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o), - PyUnicode_GET_SIZE(o), - 10); + return PyLong_FromUnicodeObject(o, 10); if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len)) return long_from_string(buffer, buffer_len); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -854,83 +854,79 @@ const char *quote_prefix = "bytearray(b"; const char *quote_postfix = ")"; Py_ssize_t length = Py_SIZE(self); - /* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */ + /* 15 == strlen(quote_prefix) + 2 + strlen(quote_postfix) + 1 */ size_t newsize; PyObject *v; - if (length > (PY_SSIZE_T_MAX - 14) / 4) { + register Py_ssize_t i; + register char c; + register char *p; + int quote; + char *test, *start; + char *buffer; + + if (length > (PY_SSIZE_T_MAX - 15) / 4) { PyErr_SetString(PyExc_OverflowError, "bytearray object is too large to make repr"); return NULL; } - newsize = 14 + 4 * length; - v = PyUnicode_FromUnicode(NULL, newsize); - if (v == NULL) { + + newsize = 15 + length * 4; + buffer = PyMem_Malloc(newsize); + if (buffer == NULL) { + PyErr_NoMemory(); return NULL; } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p; - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - { - char *test, *start; - start = PyByteArray_AS_STRING(self); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; + + /* Figure out which quote to use; single is preferred */ + quote = '\''; + start = PyByteArray_AS_STRING(self); + for (test = start; test < start+length; ++test) { + if (*test == '"') { + quote = '\''; /* back to single */ + break; } - - p = PyUnicode_AS_UNICODE(v); - while (*quote_prefix) - *p++ = *quote_prefix++; - *p++ = quote; - - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = self->ob_bytes[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c == 0) - *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; + else if (*test == '\'') + quote = '"'; + } + + p = buffer; + while (*quote_prefix) + *p++ = *quote_prefix++; + *p++ = quote; + + for (i = 0; i < length; i++) { + /* There's at least enough room for a hex escape + and a closing quote. */ + assert(newsize - (p - buffer) >= 5); + c = self->ob_bytes[i]; + if (c == '\'' || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c == 0) + *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigits[(c & 0xf0) >> 4]; + *p++ = hexdigits[c & 0xf]; } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - while (*quote_postfix) { - *p++ = *quote_postfix++; - } - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; - } - return v; + else + *p++ = c; } + assert(newsize - (p - buffer) >= 1); + *p++ = quote; + while (*quote_postfix) { + *p++ = *quote_postfix++; + } + + v = PyUnicode_DecodeASCII(buffer, p - buffer, NULL); + PyMem_Free(buffer); + return v; } static PyObject * @@ -1034,6 +1030,8 @@ /* -------------------------------------------------------------------- */ /* Methods */ +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_CHAR char #define STRINGLIB_LEN PyByteArray_GET_SIZE #define STRINGLIB_STR PyByteArray_AS_STRING @@ -2651,15 +2649,20 @@ { PyObject *newbytes, *hexobj; char *buf; - Py_UNICODE *hex; Py_ssize_t hexlen, byteslen, i, j; int top, bot; + void *data; + unsigned int kind; if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) return NULL; assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); + if (PyUnicode_READY(hexobj)) + return NULL; + kind = PyUnicode_KIND(hexobj); + data = PyUnicode_DATA(hexobj); + hexlen = PyUnicode_GET_LENGTH(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); if (!newbytes) @@ -2667,12 +2670,12 @@ buf = PyByteArray_AS_STRING(newbytes); for (i = j = 0; i < hexlen; i += 2) { /* skip over spaces in the input */ - while (hex[i] == ' ') + while (PyUnicode_READ(kind, data, i) == ' ') i++; if (i >= hexlen) break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); + top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); + bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); if (top == -1 || bot == -1) { PyErr_Format(PyExc_ValueError, "non-hexadecimal number found in " diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -566,74 +566,68 @@ { static const char *hexdigits = "0123456789abcdef"; register PyBytesObject* op = (PyBytesObject*) obj; - Py_ssize_t length = Py_SIZE(op); - size_t newsize; + Py_ssize_t i, length = Py_SIZE(op); + size_t newsize, squotes, dquotes; PyObject *v; - if (length > (PY_SSIZE_T_MAX - 3) / 4) { + unsigned char quote, *s, *p; + + /* Compute size of output string */ + squotes = dquotes = 0; + newsize = 3; /* b'' */ + s = (unsigned char*)op->ob_sval; + for (i = 0; i < length; i++) { + switch(s[i]) { + case '\'': squotes++; newsize++; break; + case '"': dquotes++; newsize++; break; + case '\\': case '\t': case '\n': case '\r': + newsize += 2; break; /* \C */ + default: + if (s[i] < ' ' || s[i] >= 0x7f) + newsize += 4; /* \xHH */ + else + newsize++; + } + } + quote = '\''; + if (smartquotes && squotes && !dquotes) + quote = '"'; + if (squotes && quote == '\'') + newsize += squotes; + + if (newsize > (PY_SSIZE_T_MAX - sizeof(PyUnicodeObject) - 1)) { PyErr_SetString(PyExc_OverflowError, "bytes object is too large to make repr"); return NULL; } - newsize = 3 + 4 * length; - v = PyUnicode_FromUnicode(NULL, newsize); + + v = PyUnicode_New(newsize, 127); if (v == NULL) { return NULL; } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p = PyUnicode_AS_UNICODE(v); - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - if (smartquotes) { - char *test, *start; - start = PyBytes_AS_STRING(op); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; + p = PyUnicode_1BYTE_DATA(v); + + *p++ = 'b', *p++ = quote; + for (i = 0; i < length; i++) { + unsigned char c = op->ob_sval[i]; + if (c == quote || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigits[(c & 0xf0) >> 4]; + *p++ = hexdigits[c & 0xf]; } - - *p++ = 'b', *p++ = quote; - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = op->ob_sval[i]; - if (c == quote || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; - } - return v; + else + *p++ = c; } + *p++ = quote; + return v; } static PyObject * @@ -2356,15 +2350,20 @@ { PyObject *newstring, *hexobj; char *buf; - Py_UNICODE *hex; Py_ssize_t hexlen, byteslen, i, j; int top, bot; + void *data; + unsigned int kind; if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) return NULL; assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); + if (PyUnicode_READY(hexobj)) + return NULL; + kind = PyUnicode_KIND(hexobj); + data = PyUnicode_DATA(hexobj); + hexlen = PyUnicode_GET_LENGTH(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ newstring = PyBytes_FromStringAndSize(NULL, byteslen); if (!newstring) @@ -2372,12 +2371,12 @@ buf = PyBytes_AS_STRING(newstring); for (i = j = 0; i < hexlen; i += 2) { /* skip over spaces in the input */ - while (hex[i] == ' ') + while (PyUnicode_READ(kind, data, i) == ' ') i++; if (i >= hexlen) break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); + top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); + bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); if (top == -1 || bot == -1) { PyErr_Format(PyExc_ValueError, "non-hexadecimal number found in " diff --git a/Objects/codeobject.c b/Objects/codeobject.c --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -8,19 +8,24 @@ /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ static int -all_name_chars(Py_UNICODE *s) +all_name_chars(PyObject *o) { static char ok_name_char[256]; static unsigned char *name_chars = (unsigned char *)NAME_CHARS; + PyUnicodeObject *u = (PyUnicodeObject *)o; + const unsigned char *s; + + if (!PyUnicode_Check(o) || PyUnicode_READY(u) == -1 || + PyUnicode_MAX_CHAR_VALUE(u) >= 128) + return 0; if (ok_name_char[*name_chars] == 0) { unsigned char *p; for (p = name_chars; *p; p++) ok_name_char[*p] = 1; } + s = PyUnicode_1BYTE_DATA(u); while (*s) { - if (*s >= 128) - return 0; if (ok_name_char[*s++] == 0) return 0; } @@ -77,9 +82,7 @@ /* Intern selected string constants */ for (i = PyTuple_GET_SIZE(consts); --i >= 0; ) { PyObject *v = PyTuple_GetItem(consts, i); - if (!PyUnicode_Check(v)) - continue; - if (!all_name_chars(PyUnicode_AS_UNICODE(v))) + if (!all_name_chars(v)) continue; PyUnicode_InternInPlace(&PyTuple_GET_ITEM(consts, i)); } diff --git a/Objects/complexobject.c b/Objects/complexobject.c --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -702,9 +702,8 @@ if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyComplex_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + return _PyComplex_FormatAdvanced(self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); } #if 0 @@ -755,20 +754,10 @@ Py_ssize_t len; if (PyUnicode_Check(v)) { - Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v); - Py_UNICODE *bufptr; - s_buffer = PyUnicode_TransformDecimalToASCII( - PyUnicode_AS_UNICODE(v), buflen); + s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); if (s_buffer == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - bufptr = PyUnicode_AS_UNICODE(s_buffer); - for (i = 0; i < buflen; i++) { - Py_UNICODE ch = bufptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - bufptr[i] = ' '; - } - s = _PyUnicode_AsStringAndSize(s_buffer, &len); + s = PyUnicode_AsUTF8AndSize(s_buffer, &len); if (s == NULL) goto error; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -710,7 +710,7 @@ if (!PyDict_Check(op)) return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) { @@ -762,7 +762,7 @@ return NULL; } if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) { @@ -797,7 +797,7 @@ assert(value); mp = (PyDictObject *)op; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) @@ -842,7 +842,7 @@ } assert(key); if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -1122,7 +1122,7 @@ PyDictEntry *ep; assert(mp->ma_table != NULL); if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; @@ -1726,7 +1726,7 @@ PyDictEntry *ep; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; @@ -1750,7 +1750,7 @@ return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; @@ -1779,7 +1779,7 @@ return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; @@ -1824,7 +1824,7 @@ return NULL; } if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; @@ -2033,7 +2033,7 @@ PyDictEntry *ep; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; diff --git a/Objects/exceptions.c b/Objects/exceptions.c --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -962,21 +962,18 @@ static PyObject* my_basename(PyObject *name) { - Py_UNICODE *unicode; Py_ssize_t i, size, offset; - - unicode = PyUnicode_AS_UNICODE(name); - size = PyUnicode_GET_SIZE(name); + int kind = PyUnicode_KIND(name); + void *data = PyUnicode_DATA(name); + size = PyUnicode_GET_LENGTH(name); offset = 0; for(i=0; i < size; i++) { - if (unicode[i] == SEP) + if (PyUnicode_READ(kind, data, i) == SEP) offset = i + 1; } - if (offset != 0) { - return PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(name) + offset, - size - offset); - } else { + if (offset != 0) + return PyUnicode_Substring(name, offset, size); + else { Py_INCREF(name); return name; } @@ -1712,6 +1709,7 @@ }; PyObject *PyExc_UnicodeTranslateError = (PyObject *)&_PyExc_UnicodeTranslateError; +/* Deprecated. */ PyObject * PyUnicodeTranslateError_Create( const Py_UNICODE *object, Py_ssize_t length, @@ -1721,6 +1719,14 @@ object, length, start, end, reason); } +PyObject * +_PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, Py_ssize_t end, const char *reason) +{ + return PyObject_CallFunction(PyExc_UnicodeTranslateError, "Ons", + object, start, end, reason); +} /* * AssertionError extends Exception diff --git a/Objects/fileobject.c b/Objects/fileobject.c --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -103,23 +103,18 @@ } } if (n < 0 && result != NULL && PyUnicode_Check(result)) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(result); - Py_ssize_t len = PyUnicode_GET_SIZE(result); + Py_ssize_t len = PyUnicode_GET_LENGTH(result); if (len == 0) { Py_DECREF(result); result = NULL; PyErr_SetString(PyExc_EOFError, "EOF when reading a line"); } - else if (s[len-1] == '\n') { - if (result->ob_refcnt == 1) - PyUnicode_Resize(&result, len-1); - else { - PyObject *v; - v = PyUnicode_FromUnicode(s, len-1); - Py_DECREF(result); - result = v; - } + else if (PyUnicode_READ_CHAR(result, len-1) == '\n') { + PyObject *v; + v = PyUnicode_Substring(result, 0, len-1); + Py_DECREF(result); + result = v; } } return result; diff --git a/Objects/floatobject.c b/Objects/floatobject.c --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -174,20 +174,10 @@ PyObject *result = NULL; if (PyUnicode_Check(v)) { - Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v); - Py_UNICODE *bufptr; - s_buffer = PyUnicode_TransformDecimalToASCII( - PyUnicode_AS_UNICODE(v), buflen); + s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); if (s_buffer == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - bufptr = PyUnicode_AS_UNICODE(s_buffer); - for (i = 0; i < buflen; i++) { - Py_UNICODE ch = bufptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - bufptr[i] = ' '; - } - s = _PyUnicode_AsStringAndSize(s_buffer, &len); + s = PyUnicode_AsUTF8AndSize(s_buffer, &len); if (s == NULL) { Py_DECREF(s_buffer); return NULL; @@ -1741,9 +1731,8 @@ if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyFloat_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + return _PyFloat_FormatAdvanced(self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); } PyDoc_STRVAR(float__format__doc, diff --git a/Objects/longobject.c b/Objects/longobject.c --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1551,7 +1551,7 @@ PyObject *str; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; - Py_UNICODE *p; + unsigned char *p; int negative; a = (PyLongObject *)aa; @@ -1619,14 +1619,15 @@ tenpow *= 10; strlen++; } - str = PyUnicode_FromUnicode(NULL, strlen); + str = PyUnicode_New(strlen, '9'); if (str == NULL) { Py_DECREF(scratch); return NULL; } /* fill the string right-to-left */ - p = PyUnicode_AS_UNICODE(str) + strlen; + assert(PyUnicode_KIND(str) == PyUnicode_1BYTE_KIND); + p = PyUnicode_1BYTE_DATA(str) + strlen; *p = '\0'; /* pout[0] through pout[size-2] contribute exactly _PyLong_DECIMAL_SHIFT digits each */ @@ -1649,7 +1650,7 @@ *--p = '-'; /* check we've counted correctly */ - assert(p == PyUnicode_AS_UNICODE(str)); + assert(p == PyUnicode_1BYTE_DATA(str)); Py_DECREF(scratch); return (PyObject *)str; } @@ -1662,10 +1663,12 @@ _PyLong_Format(PyObject *aa, int base) { register PyLongObject *a = (PyLongObject *)aa; - PyObject *str; + PyObject *v; Py_ssize_t i, sz; Py_ssize_t size_a; - Py_UNICODE *p, sign = '\0'; + char *p; + char sign = '\0'; + char *buffer; int bits; assert(base == 2 || base == 8 || base == 10 || base == 16); @@ -1695,7 +1698,7 @@ } /* compute length of output string: allow 2 characters for prefix and 1 for possible '-' sign. */ - if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) { + if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT / sizeof(Py_UCS4)) { PyErr_SetString(PyExc_OverflowError, "int is too large to format"); return NULL; @@ -1704,11 +1707,12 @@ is safe from overflow */ sz = 3 + (size_a * PyLong_SHIFT + (bits - 1)) / bits; assert(sz >= 0); - str = PyUnicode_FromUnicode(NULL, sz); - if (str == NULL) + buffer = PyMem_Malloc(sz); + if (buffer == NULL) { + PyErr_NoMemory(); return NULL; - p = PyUnicode_AS_UNICODE(str) + sz; - *p = '\0'; + } + p = &buffer[sz]; if (Py_SIZE(a) < 0) sign = '-'; @@ -1724,10 +1728,10 @@ accumbits += PyLong_SHIFT; assert(accumbits >= bits); do { - Py_UNICODE cdigit; - cdigit = (Py_UNICODE)(accum & (base - 1)); + char cdigit; + cdigit = (char)(accum & (base - 1)); cdigit += (cdigit < 10) ? '0' : 'a'-10; - assert(p > PyUnicode_AS_UNICODE(str)); + assert(p > buffer); *--p = cdigit; accumbits -= bits; accum >>= bits; @@ -1744,19 +1748,9 @@ *--p = '0'; if (sign) *--p = sign; - if (p != PyUnicode_AS_UNICODE(str)) { - Py_UNICODE *q = PyUnicode_AS_UNICODE(str); - assert(p > q); - do { - } while ((*q++ = *p++) != '\0'); - q--; - if (PyUnicode_Resize(&str,(Py_ssize_t) (q - - PyUnicode_AS_UNICODE(str)))) { - Py_DECREF(str); - return NULL; - } - } - return (PyObject *)str; + v = PyUnicode_DecodeASCII(p, &buffer[sz] - p, NULL); + PyMem_Free(buffer); + return v; } /* Table of digit values for 8-bit string -> integer conversion. @@ -2134,23 +2128,26 @@ PyObject * PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) { + PyObject *v, *unicode = PyUnicode_FromUnicode(u, length); + if (unicode == NULL) + return NULL; + v = PyLong_FromUnicodeObject(unicode, base); + Py_DECREF(unicode); + return v; +} + +PyObject * +PyLong_FromUnicodeObject(PyObject *u, int base) +{ PyObject *result; PyObject *asciidig; char *buffer, *end; - Py_ssize_t i, buflen; - Py_UNICODE *ptr; - - asciidig = PyUnicode_TransformDecimalToASCII(u, length); + Py_ssize_t buflen; + + asciidig = _PyUnicode_TransformDecimalAndSpaceToASCII(u); if (asciidig == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - ptr = PyUnicode_AS_UNICODE(asciidig); - for (i = 0; i < length; i++) { - Py_UNICODE ch = ptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - ptr[i] = ' '; - } - buffer = _PyUnicode_AsStringAndSize(asciidig, &buflen); + buffer = PyUnicode_AsUTF8AndSize(asciidig, &buflen); if (buffer == NULL) { Py_DECREF(asciidig); return NULL; @@ -4144,9 +4141,7 @@ } if (PyUnicode_Check(x)) - return PyLong_FromUnicode(PyUnicode_AS_UNICODE(x), - PyUnicode_GET_SIZE(x), - (int)base); + return PyLong_FromUnicodeObject(x, (int)base); else if (PyByteArray_Check(x) || PyBytes_Check(x)) { /* Since PyLong_FromString doesn't have a length parameter, * check here for possible NULs in the string. */ @@ -4228,9 +4223,8 @@ if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyLong_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + return _PyLong_FormatAdvanced(self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); } /* Return a pair (q, r) such that a = b * q + r, and diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -285,8 +285,8 @@ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { if (value != Py_None && PyUnicode_Check(key)) { - Py_UNICODE *u = PyUnicode_AS_UNICODE(key); - if (u[0] == '_' && u[1] != '_') { + if (PyUnicode_READ_CHAR(key, 0) == '_' && + PyUnicode_READ_CHAR(key, 1) != '_') { if (Py_VerboseFlag > 1) { const char *s = _PyUnicode_AsString(key); if (s != NULL) @@ -303,9 +303,8 @@ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { if (value != Py_None && PyUnicode_Check(key)) { - Py_UNICODE *u = PyUnicode_AS_UNICODE(key); - if (u[0] != '_' - || PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0) + if (PyUnicode_READ_CHAR(key, 0) != '_' || + PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0) { if (Py_VerboseFlag > 1) { const char *s = _PyUnicode_AsString(key); diff --git a/Objects/object.c b/Objects/object.c --- a/Objects/object.c +++ b/Objects/object.c @@ -295,9 +295,7 @@ } else if (PyUnicode_Check(s)) { PyObject *t; - t = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(s), - PyUnicode_GET_SIZE(s), - "backslashreplace"); + t = PyUnicode_AsEncodedString(s, "utf-8", "backslashreplace"); if (t == NULL) ret = 0; else { @@ -439,11 +437,7 @@ return NULL; /* repr is guaranteed to be a PyUnicode object by PyObject_Repr */ - ascii = PyUnicode_EncodeASCII( - PyUnicode_AS_UNICODE(repr), - PyUnicode_GET_SIZE(repr), - "backslashreplace"); - + ascii = _PyUnicode_AsASCIIString(repr, "backslashreplace"); Py_DECREF(repr); if (ascii == NULL) return NULL; diff --git a/Objects/setobject.c b/Objects/setobject.c --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -386,7 +386,7 @@ register Py_ssize_t n_used; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -434,7 +434,7 @@ assert (PyAnySet_Check(so)); if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -579,11 +579,8 @@ static PyObject * set_repr(PySetObject *so) { - PyObject *keys, *result=NULL; - Py_UNICODE *u; + PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); - PyObject *listrepr; - Py_ssize_t newsize; if (status != 0) { if (status < 0) @@ -601,31 +598,24 @@ if (keys == NULL) goto done; + /* repr(keys)[1:-1] */ listrepr = PyObject_Repr(keys); Py_DECREF(keys); if (listrepr == NULL) goto done; - newsize = PyUnicode_GET_SIZE(listrepr); - result = PyUnicode_FromUnicode(NULL, newsize); - if (result == NULL) + tmp = PyUnicode_Substring(listrepr, 1, PyUnicode_GET_LENGTH(listrepr)-1); + Py_DECREF(listrepr); + if (tmp == NULL) goto done; + listrepr = tmp; - u = PyUnicode_AS_UNICODE(result); - *u++ = '{'; - /* Omit the brackets from the listrepr */ - Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, - newsize-2); - u += newsize-2; - *u++ = '}'; + if (Py_TYPE(so) != &PySet_Type) + result = PyUnicode_FromFormat("%s({%U})", + Py_TYPE(so)->tp_name, + listrepr); + else + result = PyUnicode_FromFormat("{%U}", listrepr); Py_DECREF(listrepr); - - if (Py_TYPE(so) != &PySet_Type) { - PyObject *tmp = PyUnicode_FromFormat("%s(%U)", - Py_TYPE(so)->tp_name, - result); - Py_DECREF(result); - result = tmp; - } done: Py_ReprLeave((PyObject*)so); return result; @@ -684,7 +674,7 @@ setentry *entry; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; diff --git a/Objects/stringlib/count.h b/Objects/stringlib/count.h --- a/Objects/stringlib/count.h +++ b/Objects/stringlib/count.h @@ -1,14 +1,11 @@ /* stringlib: count implementation */ -#ifndef STRINGLIB_COUNT_H -#define STRINGLIB_COUNT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(count)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t maxcount) { @@ -19,7 +16,7 @@ if (sub_len == 0) return (str_len < maxcount) ? str_len + 1 : maxcount; - count = fastsearch(str, str_len, sub, sub_len, maxcount, FAST_COUNT); + count = FASTSEARCH(str, str_len, sub, sub_len, maxcount, FAST_COUNT); if (count < 0) return 0; /* no match */ @@ -27,4 +24,4 @@ return count; } -#endif + diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -9,13 +9,26 @@ register PyUnicodeObject *a = (PyUnicodeObject *)aa; register PyUnicodeObject *b = (PyUnicodeObject *)bb; - if (a->length != b->length) + if (PyUnicode_READY(a) == -1 || PyUnicode_READY(b) == -1) { + assert(0 && "unicode_eq ready fail"); return 0; - if (a->length == 0) + } + + if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + return 0; + if (PyUnicode_GET_LENGTH(a) == 0) return 1; - if (a->str[0] != b->str[0]) + if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) return 0; - if (a->length == 1) + /* Just comparing the first byte is enough to see if a and b differ. + * If they are 2 byte or 4 byte character most differences will happen in + * the lower bytes anyways. + */ + if (PyUnicode_1BYTE_DATA(a)[0] != PyUnicode_1BYTE_DATA(b)[0]) + return 0; + if (PyUnicode_KIND(a) == PyUnicode_1BYTE_KIND && + PyUnicode_GET_LENGTH(a) == 1) return 1; - return memcmp(a->str, b->str, a->length * sizeof(Py_UNICODE)) == 0; + return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), + PyUnicode_GET_LENGTH(a) * PyUnicode_CHARACTER_SIZE(a)) == 0; } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -1,6 +1,5 @@ /* stringlib: fastsearch implementation */ -#ifndef STRINGLIB_FASTSEARCH_H #define STRINGLIB_FASTSEARCH_H /* fast search/count implementation, based on a mix between boyer- @@ -34,7 +33,7 @@ ((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1))))) Py_LOCAL_INLINE(Py_ssize_t) -fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, +FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, Py_ssize_t maxcount, int mode) { @@ -157,4 +156,3 @@ return count; } -#endif diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -1,14 +1,11 @@ /* stringlib: find/index implementation */ -#ifndef STRINGLIB_FIND_H -#define STRINGLIB_FIND_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -19,7 +16,7 @@ if (sub_len == 0) return offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_SEARCH); if (pos >= 0) pos += offset; @@ -28,7 +25,7 @@ } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -39,7 +36,7 @@ if (sub_len == 0) return str_len + offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_RSEARCH); if (pos >= 0) pos += offset; @@ -63,29 +60,29 @@ } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_find(str + start, end - start, sub, sub_len, start); + return STRINGLIB(find)(str + start, end - start, sub, sub_len, start); } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_rfind(str + start, end - start, sub, sub_len, start); + return STRINGLIB(rfind)(str + start, end - start, sub, sub_len, start); } #ifdef STRINGLIB_WANT_CONTAINS_OBJ Py_LOCAL_INLINE(int) -stringlib_contains_obj(PyObject* str, PyObject* sub) +STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) { - return stringlib_find( + return STRINGLIB(find)( STRINGLIB_STR(str), STRINGLIB_LEN(str), STRINGLIB_STR(sub), STRINGLIB_LEN(sub), 0 ) != -1; @@ -105,7 +102,7 @@ #define FORMAT_BUFFER_SIZE 50 Py_LOCAL_INLINE(int) -stringlib_parse_args_finds(const char * function_name, PyObject *args, +STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, PyObject **subobj, Py_ssize_t *start, Py_ssize_t *end) { @@ -153,13 +150,13 @@ */ Py_LOCAL_INLINE(int) -stringlib_parse_args_finds_unicode(const char * function_name, PyObject *args, +STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, PyUnicodeObject **substring, Py_ssize_t *start, Py_ssize_t *end) { PyObject *tmp_substring; - if(stringlib_parse_args_finds(function_name, args, &tmp_substring, + if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring, start, end)) { tmp_substring = PyUnicode_FromObject(tmp_substring); if (!tmp_substring) @@ -171,5 +168,3 @@ } #endif /* STRINGLIB_IS_UNICODE */ - -#endif /* STRINGLIB_FIND_H */ diff --git a/Objects/stringlib/formatter.h b/Objects/stringlib/formatter.h deleted file mode 100644 --- a/Objects/stringlib/formatter.h +++ /dev/null @@ -1,1516 +0,0 @@ -/* implements the string, long, and float formatters. that is, - string.__format__, etc. */ - -#include <locale.h> - -/* Before including this, you must include either: - stringlib/unicodedefs.h - stringlib/stringdefs.h - - Also, you should define the names: - FORMAT_STRING - FORMAT_LONG - FORMAT_FLOAT - FORMAT_COMPLEX - to be whatever you want the public names of these functions to - be. These are the only non-static functions defined here. -*/ - -/* Raises an exception about an unknown presentation type for this - * type. */ - -static void -unknown_presentation_type(STRINGLIB_CHAR presentation_type, - const char* type_name) -{ -#if STRINGLIB_IS_UNICODE - /* If STRINGLIB_CHAR is Py_UNICODE, %c might be out-of-range, - hence the two cases. If it is char, gcc complains that the - condition below is always true, hence the ifdef. */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Unknown format code '%c' " - "for object of type '%.200s'", - (char)presentation_type, - type_name); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Unknown format code '\\x%x' " - "for object of type '%.200s'", - (unsigned int)presentation_type, - type_name); -#endif -} - -static void -invalid_comma_type(STRINGLIB_CHAR presentation_type) -{ -#if STRINGLIB_IS_UNICODE - /* See comment in unknown_presentation_type */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '%c'.", - (char)presentation_type); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '\\x%x'.", - (unsigned int)presentation_type); -#endif -} - -/* - get_integer consumes 0 or more decimal digit characters from an - input string, updates *result with the corresponding positive - integer, and returns the number of digits consumed. - - returns -1 on error. -*/ -static int -get_integer(STRINGLIB_CHAR **ptr, STRINGLIB_CHAR *end, - Py_ssize_t *result) -{ - Py_ssize_t accumulator, digitval; - int numdigits; - accumulator = numdigits = 0; - for (;;(*ptr)++, numdigits++) { - if (*ptr >= end) - break; - digitval = STRINGLIB_TODECIMAL(**ptr); - if (digitval < 0) - break; - /* - Detect possible overflow before it happens: - - accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if - accumulator > (PY_SSIZE_T_MAX - digitval) / 10. - */ - if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { - PyErr_Format(PyExc_ValueError, - "Too many decimal digits in format string"); - return -1; - } - accumulator = accumulator * 10 + digitval; - } - *result = accumulator; - return numdigits; -} - -/************************************************************************/ -/*********** standard format specifier parsing **************************/ -/************************************************************************/ - -/* returns true if this character is a specifier alignment token */ -Py_LOCAL_INLINE(int) -is_alignment_token(STRINGLIB_CHAR c) -{ - switch (c) { - case '<': case '>': case '=': case '^': - return 1; - default: - return 0; - } -} - -/* returns true if this character is a sign element */ -Py_LOCAL_INLINE(int) -is_sign_element(STRINGLIB_CHAR c) -{ - switch (c) { - case ' ': case '+': case '-': - return 1; - default: - return 0; - } -} - - -typedef struct { - STRINGLIB_CHAR fill_char; - STRINGLIB_CHAR align; - int alternate; - STRINGLIB_CHAR sign; - Py_ssize_t width; - int thousands_separators; - Py_ssize_t precision; - STRINGLIB_CHAR type; -} InternalFormatSpec; - - -#if 0 -/* Occassionally useful for debugging. Should normally be commented out. */ -static void -DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format) -{ - printf("internal format spec: fill_char %d\n", format->fill_char); - printf("internal format spec: align %d\n", format->align); - printf("internal format spec: alternate %d\n", format->alternate); - printf("internal format spec: sign %d\n", format->sign); - printf("internal format spec: width %zd\n", format->width); - printf("internal format spec: thousands_separators %d\n", - format->thousands_separators); - printf("internal format spec: precision %zd\n", format->precision); - printf("internal format spec: type %c\n", format->type); - printf("\n"); -} -#endif - - -/* - ptr points to the start of the format_spec, end points just past its end. - fills in format with the parsed information. - returns 1 on success, 0 on failure. - if failure, sets the exception -*/ -static int -parse_internal_render_format_spec(STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - InternalFormatSpec *format, - char default_type, - char default_align) -{ - STRINGLIB_CHAR *ptr = format_spec; - STRINGLIB_CHAR *end = format_spec + format_spec_len; - - /* end-ptr is used throughout this code to specify the length of - the input string */ - - Py_ssize_t consumed; - int align_specified = 0; - - format->fill_char = '\0'; - format->align = default_align; - format->alternate = 0; - format->sign = '\0'; - format->width = -1; - format->thousands_separators = 0; - format->precision = -1; - format->type = default_type; - - /* If the second char is an alignment token, - then parse the fill char */ - if (end-ptr >= 2 && is_alignment_token(ptr[1])) { - format->align = ptr[1]; - format->fill_char = ptr[0]; - align_specified = 1; - ptr += 2; - } - else if (end-ptr >= 1 && is_alignment_token(ptr[0])) { - format->align = ptr[0]; - align_specified = 1; - ++ptr; - } - - /* Parse the various sign options */ - if (end-ptr >= 1 && is_sign_element(ptr[0])) { - format->sign = ptr[0]; - ++ptr; - } - - /* If the next character is #, we're in alternate mode. This only - applies to integers. */ - if (end-ptr >= 1 && ptr[0] == '#') { - format->alternate = 1; - ++ptr; - } - - /* The special case for 0-padding (backwards compat) */ - if (format->fill_char == '\0' && end-ptr >= 1 && ptr[0] == '0') { - format->fill_char = '0'; - if (!align_specified) { - format->align = '='; - } - ++ptr; - } - - consumed = get_integer(&ptr, end, &format->width); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* If consumed is 0, we didn't consume any characters for the - width. In that case, reset the width to -1, because - get_integer() will have set it to zero. -1 is how we record - that the width wasn't specified. */ - if (consumed == 0) - format->width = -1; - - /* Comma signifies add thousands separators */ - if (end-ptr && ptr[0] == ',') { - format->thousands_separators = 1; - ++ptr; - } - - /* Parse field precision */ - if (end-ptr && ptr[0] == '.') { - ++ptr; - - consumed = get_integer(&ptr, end, &format->precision); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* Not having a precision after a dot is an error. */ - if (consumed == 0) { - PyErr_Format(PyExc_ValueError, - "Format specifier missing precision"); - return 0; - } - - } - - /* Finally, parse the type field. */ - - if (end-ptr > 1) { - /* More than one char remain, invalid conversion spec. */ - PyErr_Format(PyExc_ValueError, "Invalid conversion specification"); - return 0; - } - - if (end-ptr == 1) { - format->type = ptr[0]; - ++ptr; - } - - /* Do as much validating as we can, just by looking at the format - specifier. Do not take into account what type of formatting - we're doing (int, float, string). */ - - if (format->thousands_separators) { - switch (format->type) { - case 'd': - case 'e': - case 'f': - case 'g': - case 'E': - case 'G': - case '%': - case 'F': - case '\0': - /* These are allowed. See PEP 378.*/ - break; - default: - invalid_comma_type(format->type); - return 0; - } - } - - return 1; -} - -/* Calculate the padding needed. */ -static void -calc_padding(Py_ssize_t nchars, Py_ssize_t width, STRINGLIB_CHAR align, - Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding, - Py_ssize_t *n_total) -{ - if (width >= 0) { - if (nchars > width) - *n_total = nchars; - else - *n_total = width; - } - else { - /* not specified, use all of the chars and no more */ - *n_total = nchars; - } - - /* Figure out how much leading space we need, based on the - aligning */ - if (align == '>') - *n_lpadding = *n_total - nchars; - else if (align == '^') - *n_lpadding = (*n_total - nchars) / 2; - else if (align == '<' || align == '=') - *n_lpadding = 0; - else { - /* We should never have an unspecified alignment. */ - *n_lpadding = 0; - assert(0); - } - - *n_rpadding = *n_total - nchars - *n_lpadding; -} - -/* Do the padding, and return a pointer to where the caller-supplied - content goes. */ -static STRINGLIB_CHAR * -fill_padding(STRINGLIB_CHAR *p, Py_ssize_t nchars, STRINGLIB_CHAR fill_char, - Py_ssize_t n_lpadding, Py_ssize_t n_rpadding) -{ - /* Pad on left. */ - if (n_lpadding) - STRINGLIB_FILL(p, fill_char, n_lpadding); - - /* Pad on right. */ - if (n_rpadding) - STRINGLIB_FILL(p + nchars + n_lpadding, fill_char, n_rpadding); - - /* Pointer to the user content. */ - return p + n_lpadding; -} - -#if defined FORMAT_FLOAT || defined FORMAT_LONG || defined FORMAT_COMPLEX -/************************************************************************/ -/*********** common routines for numeric formatting *********************/ -/************************************************************************/ - -/* Locale type codes. */ -#define LT_CURRENT_LOCALE 0 -#define LT_DEFAULT_LOCALE 1 -#define LT_NO_LOCALE 2 - -/* Locale info needed for formatting integers and the part of floats - before and including the decimal. Note that locales only support - 8-bit chars, not unicode. */ -typedef struct { - char *decimal_point; - char *thousands_sep; - char *grouping; -} LocaleInfo; - -/* describes the layout for an integer, see the comment in - calc_number_widths() for details */ -typedef struct { - Py_ssize_t n_lpadding; - Py_ssize_t n_prefix; - Py_ssize_t n_spadding; - Py_ssize_t n_rpadding; - char sign; - Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */ - Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including - any grouping chars. */ - Py_ssize_t n_decimal; /* 0 if only an integer */ - Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part, - excluding the decimal itself, if - present. */ - - /* These 2 are not the widths of fields, but are needed by - STRINGLIB_GROUPING. */ - Py_ssize_t n_digits; /* The number of digits before a decimal - or exponent. */ - Py_ssize_t n_min_width; /* The min_width we used when we computed - the n_grouped_digits width. */ -} NumberFieldWidths; - - -/* Given a number of the form: - digits[remainder] - where ptr points to the start and end points to the end, find where - the integer part ends. This could be a decimal, an exponent, both, - or neither. - If a decimal point is present, set *has_decimal and increment - remainder beyond it. - Results are undefined (but shouldn't crash) for improperly - formatted strings. -*/ -static void -parse_number(STRINGLIB_CHAR *ptr, Py_ssize_t len, - Py_ssize_t *n_remainder, int *has_decimal) -{ - STRINGLIB_CHAR *end = ptr + len; - STRINGLIB_CHAR *remainder; - - while (ptr<end && isdigit(*ptr)) - ++ptr; - remainder = ptr; - - /* Does remainder start with a decimal point? */ - *has_decimal = ptr<end && *remainder == '.'; - - /* Skip the decimal point. */ - if (*has_decimal) - remainder++; - - *n_remainder = end - remainder; -} - -/* not all fields of format are used. for example, precision is - unused. should this take discrete params in order to be more clear - about what it does? or is passing a single format parameter easier - and more efficient enough to justify a little obfuscation? */ -static Py_ssize_t -calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, - STRINGLIB_CHAR sign_char, STRINGLIB_CHAR *number, - Py_ssize_t n_number, Py_ssize_t n_remainder, - int has_decimal, const LocaleInfo *locale, - const InternalFormatSpec *format) -{ - Py_ssize_t n_non_digit_non_padding; - Py_ssize_t n_padding; - - spec->n_digits = n_number - n_remainder - (has_decimal?1:0); - spec->n_lpadding = 0; - spec->n_prefix = n_prefix; - spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0; - spec->n_remainder = n_remainder; - spec->n_spadding = 0; - spec->n_rpadding = 0; - spec->sign = '\0'; - spec->n_sign = 0; - - /* the output will look like: - | | - | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> | - | | - - sign is computed from format->sign and the actual - sign of the number - - prefix is given (it's for the '0x' prefix) - - digits is already known - - the total width is either given, or computed from the - actual digits - - only one of lpadding, spadding, and rpadding can be non-zero, - and it's calculated from the width and other fields - */ - - /* compute the various parts we're going to write */ - switch (format->sign) { - case '+': - /* always put a + or - */ - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : '+'); - break; - case ' ': - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : ' '); - break; - default: - /* Not specified, or the default (-) */ - if (sign_char == '-') { - spec->n_sign = 1; - spec->sign = '-'; - } - } - - /* The number of chars used for non-digits and non-padding. */ - n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal + - spec->n_remainder; - - /* min_width can go negative, that's okay. format->width == -1 means - we don't care. */ - if (format->fill_char == '0' && format->align == '=') - spec->n_min_width = format->width - n_non_digit_non_padding; - else - spec->n_min_width = 0; - - if (spec->n_digits == 0) - /* This case only occurs when using 'c' formatting, we need - to special case it because the grouping code always wants - to have at least one character. */ - spec->n_grouped_digits = 0; - else - spec->n_grouped_digits = STRINGLIB_GROUPING(NULL, 0, NULL, - spec->n_digits, - spec->n_min_width, - locale->grouping, - locale->thousands_sep); - - /* Given the desired width and the total of digit and non-digit - space we consume, see if we need any padding. format->width can - be negative (meaning no padding), but this code still works in - that case. */ - n_padding = format->width - - (n_non_digit_non_padding + spec->n_grouped_digits); - if (n_padding > 0) { - /* Some padding is needed. Determine if it's left, space, or right. */ - switch (format->align) { - case '<': - spec->n_rpadding = n_padding; - break; - case '^': - spec->n_lpadding = n_padding / 2; - spec->n_rpadding = n_padding - spec->n_lpadding; - break; - case '=': - spec->n_spadding = n_padding; - break; - case '>': - spec->n_lpadding = n_padding; - break; - default: - /* Shouldn't get here, but treat it as '>' */ - spec->n_lpadding = n_padding; - assert(0); - break; - } - } - return spec->n_lpadding + spec->n_sign + spec->n_prefix + - spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + - spec->n_remainder + spec->n_rpadding; -} - -/* Fill in the digit parts of a numbers's string representation, - as determined in calc_number_widths(). - No error checking, since we know the buffer is the correct size. */ -static void -fill_number(STRINGLIB_CHAR *buf, const NumberFieldWidths *spec, - STRINGLIB_CHAR *digits, Py_ssize_t n_digits, - STRINGLIB_CHAR *prefix, STRINGLIB_CHAR fill_char, - LocaleInfo *locale, int toupper) -{ - /* Used to keep track of digits, decimal, and remainder. */ - STRINGLIB_CHAR *p = digits; - -#ifndef NDEBUG - Py_ssize_t r; -#endif - - if (spec->n_lpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_lpadding); - buf += spec->n_lpadding; - } - if (spec->n_sign == 1) { - *buf++ = spec->sign; - } - if (spec->n_prefix) { - memmove(buf, - prefix, - spec->n_prefix * sizeof(STRINGLIB_CHAR)); - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_prefix; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_prefix; - } - if (spec->n_spadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_spadding); - buf += spec->n_spadding; - } - - /* Only for type 'c' special case, it has no digits. */ - if (spec->n_digits != 0) { - /* Fill the digits with InsertThousandsGrouping. */ -#ifndef NDEBUG - r = -#endif - STRINGLIB_GROUPING(buf, spec->n_grouped_digits, digits, - spec->n_digits, spec->n_min_width, - locale->grouping, locale->thousands_sep); -#ifndef NDEBUG - assert(r == spec->n_grouped_digits); -#endif - p += spec->n_digits; - } - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_grouped_digits; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_grouped_digits; - - if (spec->n_decimal) { - Py_ssize_t t; - for (t = 0; t < spec->n_decimal; ++t) - buf[t] = locale->decimal_point[t]; - buf += spec->n_decimal; - p += 1; - } - - if (spec->n_remainder) { - memcpy(buf, p, spec->n_remainder * sizeof(STRINGLIB_CHAR)); - buf += spec->n_remainder; - p += spec->n_remainder; - } - - if (spec->n_rpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_rpadding); - buf += spec->n_rpadding; - } -} - -static char no_grouping[1] = {CHAR_MAX}; - -/* Find the decimal point character(s?), thousands_separator(s?), and - grouping description, either for the current locale if type is - LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or - none if LT_NO_LOCALE. */ -static void -get_locale_info(int type, LocaleInfo *locale_info) -{ - switch (type) { - case LT_CURRENT_LOCALE: { - struct lconv *locale_data = localeconv(); - locale_info->decimal_point = locale_data->decimal_point; - locale_info->thousands_sep = locale_data->thousands_sep; - locale_info->grouping = locale_data->grouping; - break; - } - case LT_DEFAULT_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ","; - locale_info->grouping = "\3"; /* Group every 3 characters. The - (implicit) trailing 0 means repeat - infinitely. */ - break; - case LT_NO_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ""; - locale_info->grouping = no_grouping; - break; - default: - assert(0); - } -} - -#endif /* FORMAT_FLOAT || FORMAT_LONG || FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** string formatting ******************************************/ -/************************************************************************/ - -static PyObject * -format_string_internal(PyObject *value, const InternalFormatSpec *format) -{ - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - STRINGLIB_CHAR *p; - Py_ssize_t len = STRINGLIB_LEN(value); - PyObject *result = NULL; - - /* sign is not allowed on strings */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed in string format specifier"); - goto done; - } - - /* alternate is not allowed on strings */ - if (format->alternate) { - PyErr_SetString(PyExc_ValueError, - "Alternate form (#) not allowed in string format " - "specifier"); - goto done; - } - - /* '=' alignment not allowed on strings */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment not allowed " - "in string format specifier"); - goto done; - } - - /* if precision is specified, output no more that format.precision - characters */ - if (format->precision >= 0 && len >= format->precision) { - len = format->precision; - } - - calc_padding(len, format->width, format->align, &lpad, &rpad, &total); - - /* allocate the resulting string */ - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Write into that space. First the padding. */ - p = fill_padding(STRINGLIB_STR(result), len, - format->fill_char=='\0'?' ':format->fill_char, - lpad, rpad); - - /* Then the source string. */ - memcpy(p, STRINGLIB_STR(value), len * sizeof(STRINGLIB_CHAR)); - -done: - return result; -} - - -/************************************************************************/ -/*********** long formatting ********************************************/ -/************************************************************************/ - -#if defined FORMAT_LONG || defined FORMAT_INT -typedef PyObject* -(*IntOrLongToString)(PyObject *value, int base); - -static PyObject * -format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - STRINGLIB_CHAR *pnumeric_chars; - STRINGLIB_CHAR numeric_char; - STRINGLIB_CHAR sign_char = '\0'; - Py_ssize_t n_digits; /* count of digits need from the computed - string */ - Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which - produces non-digits */ - Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */ - Py_ssize_t n_total; - STRINGLIB_CHAR *prefix = NULL; - NumberFieldWidths spec; - long x; - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* no precision allowed on integers */ - if (format->precision != -1) { - PyErr_SetString(PyExc_ValueError, - "Precision not allowed in integer format specifier"); - goto done; - } - - /* special case for character formatting */ - if (format->type == 'c') { - /* error to specify a sign */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed with integer" - " format specifier 'c'"); - goto done; - } - - /* taken from unicodeobject.c formatchar() */ - /* Integer input truncated to a character */ -/* XXX: won't work for int */ - x = PyLong_AsLong(value); - if (x == -1 && PyErr_Occurred()) - goto done; -#ifdef Py_UNICODE_WIDE - if (x < 0 || x > 0x10ffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000) " - "(wide Python build)"); - goto done; - } -#else - if (x < 0 || x > 0xffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x10000) " - "(narrow Python build)"); - goto done; - } -#endif - numeric_char = (STRINGLIB_CHAR)x; - pnumeric_chars = &numeric_char; - n_digits = 1; - - /* As a sort-of hack, we tell calc_number_widths that we only - have "remainder" characters. calc_number_widths thinks - these are characters that don't get formatted, only copied - into the output string. We do this for 'c' formatting, - because the characters are likely to be non-digits. */ - n_remainder = 1; - } - else { - int base; - int leading_chars_to_skip = 0; /* Number of characters added by - PyNumber_ToBase that we want to - skip over. */ - - /* Compute the base and how many characters will be added by - PyNumber_ToBase */ - switch (format->type) { - case 'b': - base = 2; - leading_chars_to_skip = 2; /* 0b */ - break; - case 'o': - base = 8; - leading_chars_to_skip = 2; /* 0o */ - break; - case 'x': - case 'X': - base = 16; - leading_chars_to_skip = 2; /* 0x */ - break; - default: /* shouldn't be needed, but stops a compiler warning */ - case 'd': - case 'n': - base = 10; - break; - } - - /* The number of prefix chars is the same as the leading - chars to skip */ - if (format->alternate) - n_prefix = leading_chars_to_skip; - - /* Do the hard part, converting to a string in a given base */ - tmp = tostring(value, base); - if (tmp == NULL) - goto done; - - pnumeric_chars = STRINGLIB_STR(tmp); - n_digits = STRINGLIB_LEN(tmp); - - prefix = pnumeric_chars; - - /* Remember not to modify what pnumeric_chars points to. it - might be interned. Only modify it after we copy it into a - newly allocated output buffer. */ - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (pnumeric_chars[0] == '-') { - sign_char = pnumeric_chars[0]; - ++prefix; - ++leading_chars_to_skip; - } - - /* Skip over the leading chars (0x, 0b, etc.) */ - n_digits -= leading_chars_to_skip; - pnumeric_chars += leading_chars_to_skip; - } - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, n_prefix, sign_char, pnumeric_chars, - n_digits, n_remainder, 0, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (!result) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, pnumeric_chars, n_digits, - prefix, format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* defined FORMAT_LONG || defined FORMAT_INT */ - -/************************************************************************/ -/*********** float formatting *******************************************/ -/************************************************************************/ - -#ifdef FORMAT_FLOAT -#if STRINGLIB_IS_UNICODE -static void -strtounicode(Py_UNICODE *buffer, const char *charbuffer, Py_ssize_t len) -{ - Py_ssize_t i; - for (i = 0; i < len; ++i) - buffer[i] = (Py_UNICODE)charbuffer[i]; -} -#endif - -/* much of this is taken from unicodeobject.c */ -static PyObject * -format_float_internal(PyObject *value, - const InternalFormatSpec *format) -{ - char *buf = NULL; /* buffer returned from PyOS_double_to_string */ - Py_ssize_t n_digits; - Py_ssize_t n_remainder; - Py_ssize_t n_total; - int has_decimal; - double val; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - int add_pct = 0; - STRINGLIB_CHAR *p; - NumberFieldWidths spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR sign_char = '\0'; - int float_type; /* Used to see if we have a nan, inf, or regular float. */ - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Behaves in the same way as repr(x) - and str(x) if no precision is given, else like 'g', but with - at least one digit after the decimal point. */ - flags |= Py_DTSF_ADD_DOT_0; - type = 'r'; - default_precision = 0; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - val = PyFloat_AsDouble(value); - if (val == -1.0 && PyErr_Occurred()) - goto done; - - if (type == '%') { - type = 'f'; - val *= 100; - add_pct = 1; - } - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - buf = PyOS_double_to_string(val, (char)type, precision, flags, - &float_type); - if (buf == NULL) - goto done; - n_digits = strlen(buf); - - if (add_pct) { - /* We know that buf has a trailing zero (since we just called - strlen() on it), and we don't use that fact any more. So we - can just write over the trailing zero. */ - buf[n_digits] = '%'; - n_digits += 1; - } - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_digits)*sizeof(Py_UNICODE)); - if (unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(unicode_tmp, buf, n_digits); - p = unicode_tmp; -#else - p = buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p == '-') { - sign_char = *p; - ++p; - --n_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p, n_digits, &n_remainder, &has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, 0, sign_char, p, n_digits, - n_remainder, has_decimal, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (result == NULL) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, p, n_digits, NULL, - format->fill_char == '\0' ? ' ' : format->fill_char, &locale, - 0); - -done: - PyMem_Free(buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_FLOAT */ - -/************************************************************************/ -/*********** complex formatting *****************************************/ -/************************************************************************/ - -#ifdef FORMAT_COMPLEX - -static PyObject * -format_complex_internal(PyObject *value, - const InternalFormatSpec *format) -{ - double re; - double im; - char *re_buf = NULL; /* buffer returned from PyOS_double_to_string */ - char *im_buf = NULL; /* buffer returned from PyOS_double_to_string */ - - InternalFormatSpec tmp_format = *format; - Py_ssize_t n_re_digits; - Py_ssize_t n_im_digits; - Py_ssize_t n_re_remainder; - Py_ssize_t n_im_remainder; - Py_ssize_t n_re_total; - Py_ssize_t n_im_total; - int re_has_decimal; - int im_has_decimal; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - STRINGLIB_CHAR *p_re; - STRINGLIB_CHAR *p_im; - NumberFieldWidths re_spec; - NumberFieldWidths im_spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR *p; - STRINGLIB_CHAR re_sign_char = '\0'; - STRINGLIB_CHAR im_sign_char = '\0'; - int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ - int im_float_type; - int add_parens = 0; - int skip_re = 0; - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *re_unicode_tmp = NULL; - Py_UNICODE *im_unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* Zero padding is not allowed. */ - if (format->fill_char == '0') { - PyErr_SetString(PyExc_ValueError, - "Zero padding is not allowed in complex format " - "specifier"); - goto done; - } - - /* Neither is '=' alignment . */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment flag is not allowed in complex format " - "specifier"); - goto done; - } - - re = PyComplex_RealAsDouble(value); - if (re == -1.0 && PyErr_Occurred()) - goto done; - im = PyComplex_ImagAsDouble(value); - if (im == -1.0 && PyErr_Occurred()) - goto done; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Should be like str(self). */ - type = 'r'; - default_precision = 0; - if (re == 0.0 && copysign(1.0, re) == 1.0) - skip_re = 1; - else - add_parens = 1; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - re_buf = PyOS_double_to_string(re, (char)type, precision, flags, - &re_float_type); - if (re_buf == NULL) - goto done; - im_buf = PyOS_double_to_string(im, (char)type, precision, flags, - &im_float_type); - if (im_buf == NULL) - goto done; - - n_re_digits = strlen(re_buf); - n_im_digits = strlen(im_buf); - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - re_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_re_digits)*sizeof(Py_UNICODE)); - if (re_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(re_unicode_tmp, re_buf, n_re_digits); - p_re = re_unicode_tmp; - - im_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_im_digits)*sizeof(Py_UNICODE)); - if (im_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(im_unicode_tmp, im_buf, n_im_digits); - p_im = im_unicode_tmp; -#else - p_re = re_buf; - p_im = im_buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p_re == '-') { - re_sign_char = *p_re; - ++p_re; - --n_re_digits; - } - if (*p_im == '-') { - im_sign_char = *p_im; - ++p_im; - --n_im_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p_re, n_re_digits, &n_re_remainder, &re_has_decimal); - parse_number(p_im, n_im_digits, &n_im_remainder, &im_has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Turn off any padding. We'll do it later after we've composed - the numbers without padding. */ - tmp_format.fill_char = '\0'; - tmp_format.align = '<'; - tmp_format.width = -1; - - /* Calculate how much memory we'll need. */ - n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, p_re, - n_re_digits, n_re_remainder, - re_has_decimal, &locale, &tmp_format); - - /* Same formatting, but always include a sign, unless the real part is - * going to be omitted, in which case we use whatever sign convention was - * requested by the original format. */ - if (!skip_re) - tmp_format.sign = '+'; - n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, p_im, - n_im_digits, n_im_remainder, - im_has_decimal, &locale, &tmp_format); - - if (skip_re) - n_re_total = 0; - - /* Add 1 for the 'j', and optionally 2 for parens. */ - calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, - format->width, format->align, &lpad, &rpad, &total); - - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Populate the memory. First, the padding. */ - p = fill_padding(STRINGLIB_STR(result), - n_re_total + n_im_total + 1 + add_parens * 2, - format->fill_char=='\0' ? ' ' : format->fill_char, - lpad, rpad); - - if (add_parens) - *p++ = '('; - - if (!skip_re) { - fill_number(p, &re_spec, p_re, n_re_digits, NULL, 0, &locale, 0); - p += n_re_total; - } - fill_number(p, &im_spec, p_im, n_im_digits, NULL, 0, &locale, 0); - p += n_im_total; - *p++ = 'j'; - - if (add_parens) - *p++ = ')'; - -done: - PyMem_Free(re_buf); - PyMem_Free(im_buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(re_unicode_tmp); - PyMem_Free(im_unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** built in formatters ****************************************/ -/************************************************************************/ -PyObject * -FORMAT_STRING(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - InternalFormatSpec format; - PyObject *result = NULL; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, format_spec_len, - &format, 's', '<')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 's': - /* no type conversion needed, already a string. do the formatting */ - result = format_string_internal(obj, &format); - break; - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} - -#if defined FORMAT_LONG || defined FORMAT_INT -static PyObject* -format_int_or_long(PyObject* obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, 'd', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 'b': - case 'c': - case 'd': - case 'o': - case 'x': - case 'X': - case 'n': - /* no type conversion needed, already an int (or long). do - the formatting */ - result = format_int_or_long_internal(obj, &format, tostring); - break; - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case '%': - /* convert to float */ - tmp = PyNumber_Float(obj); - if (tmp == NULL) - goto done; - result = format_float_internal(tmp, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* FORMAT_LONG || defined FORMAT_INT */ - -#ifdef FORMAT_LONG -/* Need to define long_format as a function that will convert a long - to a string. In 3.0, _PyLong_Format has the correct signature. In - 2.x, we need to fudge a few parameters */ -#if PY_VERSION_HEX >= 0x03000000 -#define long_format _PyLong_Format -#else -static PyObject* -long_format(PyObject* value, int base) -{ - /* Convert to base, don't add trailing 'L', and use the new octal - format. We already know this is a long object */ - assert(PyLong_Check(value)); - /* convert to base, don't add 'L', and use the new octal format */ - return _PyLong_Format(value, base, 0, 1); -} -#endif - -PyObject * -FORMAT_LONG(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - long_format); -} -#endif /* FORMAT_LONG */ - -#ifdef FORMAT_INT -/* this is only used for 2.x, not 3.0 */ -static PyObject* -int_format(PyObject* value, int base) -{ - /* Convert to base, and use the new octal format. We already - know this is an int object */ - assert(PyInt_Check(value)); - return _PyInt_Format((PyIntObject*)value, base, 1); -} - -PyObject * -FORMAT_INT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - int_format); -} -#endif /* FORMAT_INT */ - -#ifdef FORMAT_FLOAT -PyObject * -FORMAT_FLOAT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - case '%': - /* no conversion, already a float. do the formatting */ - result = format_float_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_FLOAT */ - -#ifdef FORMAT_COMPLEX -PyObject * -FORMAT_COMPLEX(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - /* no conversion, already a complex. do the formatting */ - result = format_complex_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_COMPLEX */ diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h --- a/Objects/stringlib/localeutil.h +++ b/Objects/stringlib/localeutil.h @@ -1,8 +1,5 @@ /* stringlib: locale related helpers implementation */ -#ifndef STRINGLIB_LOCALEUTIL_H -#define STRINGLIB_LOCALEUTIL_H - #include <locale.h> #define MAX(x, y) ((x) < (y) ? (y) : (x)) @@ -12,10 +9,10 @@ const char *grouping; char previous; Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; +} STRINGLIB(GroupGenerator); static void -_GroupGenerator_init(GroupGenerator *self, const char *grouping) +STRINGLIB(GroupGenerator_init)(STRINGLIB(GroupGenerator) *self, const char *grouping) { self->grouping = grouping; self->i = 0; @@ -24,7 +21,7 @@ /* Returns the next grouping, or 0 to signify end. */ static Py_ssize_t -_GroupGenerator_next(GroupGenerator *self) +STRINGLIB(GroupGenerator_next)(STRINGLIB(GroupGenerator) *self) { /* Note that we don't really do much error checking here. If a grouping string contains just CHAR_MAX, for example, then just @@ -48,13 +45,11 @@ /* Fill in some digits, leading zeros, and thousands separator. All are optional, depending on when we're called. */ static void -fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, +STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep, Py_ssize_t thousands_sep_len) { -#if STRINGLIB_IS_UNICODE Py_ssize_t i; -#endif if (thousands_sep) { *buffer_end -= thousands_sep_len; @@ -76,7 +71,8 @@ memcpy(*buffer_end, *digits_end, n_chars * sizeof(STRINGLIB_CHAR)); *buffer_end -= n_zeros; - STRINGLIB_FILL(*buffer_end, '0', n_zeros); + for (i = 0; i < n_zeros; i++) + (*buffer_end)[i] = '0'; } /** @@ -133,15 +129,15 @@ be looked at */ /* A generator that returns all of the grouping widths, until it returns 0. */ - GroupGenerator groupgen; - _GroupGenerator_init(&groupgen, grouping); + STRINGLIB(GroupGenerator) groupgen; + STRINGLIB(GroupGenerator_init)(&groupgen, grouping); if (buffer) { buffer_end = buffer + n_buffer; digits_end = digits + n_digits; } - while ((l = _GroupGenerator_next(&groupgen)) > 0) { + while ((l = STRINGLIB(GroupGenerator_next)(&groupgen)) > 0) { l = MIN(l, MAX(MAX(remaining, min_width), 1)); n_zeros = MAX(0, l - remaining); n_chars = MAX(0, MIN(remaining, l)); @@ -153,7 +149,7 @@ if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } @@ -180,7 +176,7 @@ count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } } @@ -209,4 +205,3 @@ return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits, min_width, grouping, thousands_sep); } -#endif /* STRINGLIB_LOCALEUTIL_H */ diff --git a/Objects/stringlib/partition.h b/Objects/stringlib/partition.h --- a/Objects/stringlib/partition.h +++ b/Objects/stringlib/partition.h @@ -1,14 +1,11 @@ /* stringlib: partition implementation */ -#ifndef STRINGLIB_PARTITION_H -#define STRINGLIB_PARTITION_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(PyObject*) -stringlib_partition(PyObject* str_obj, +STRINGLIB(partition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -25,7 +22,7 @@ if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -58,7 +55,7 @@ } Py_LOCAL_INLINE(PyObject*) -stringlib_rpartition(PyObject* str_obj, +STRINGLIB(rpartition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -75,7 +72,7 @@ if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -107,4 +104,3 @@ return out; } -#endif diff --git a/Objects/stringlib/split.h b/Objects/stringlib/split.h --- a/Objects/stringlib/split.h +++ b/Objects/stringlib/split.h @@ -1,8 +1,5 @@ /* stringlib: split implementation */ -#ifndef STRINGLIB_SPLIT_H -#define STRINGLIB_SPLIT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif @@ -54,7 +51,7 @@ #define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count Py_LOCAL_INLINE(PyObject *) -stringlib_split_whitespace(PyObject* str_obj, +STRINGLIB(split_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -102,7 +99,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_split_char(PyObject* str_obj, +STRINGLIB(split_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -145,7 +142,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_split(PyObject* str_obj, +STRINGLIB(split)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -158,7 +155,7 @@ return NULL; } else if (sep_len == 1) - return stringlib_split_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(split_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -166,7 +163,7 @@ i = j = 0; while (maxcount-- > 0) { - pos = fastsearch(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) break; j = i + pos; @@ -193,7 +190,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_whitespace(PyObject* str_obj, +STRINGLIB(rsplit_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -243,7 +240,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_char(PyObject* str_obj, +STRINGLIB(rsplit_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -287,7 +284,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit(PyObject* str_obj, +STRINGLIB(rsplit)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -300,7 +297,7 @@ return NULL; } else if (sep_len == 1) - return stringlib_rsplit_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(rsplit_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -308,7 +305,7 @@ j = str_len; while (maxcount-- > 0) { - pos = fastsearch(str, j, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, j, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) break; SPLIT_ADD(str, pos + sep_len, j); @@ -336,7 +333,7 @@ } Py_LOCAL_INLINE(PyObject *) -stringlib_splitlines(PyObject* str_obj, +STRINGLIB(splitlines)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, int keepends) { @@ -391,4 +388,3 @@ return NULL; } -#endif diff --git a/Objects/stringlib/string_format.h b/Objects/stringlib/string_format.h deleted file mode 100644 --- a/Objects/stringlib/string_format.h +++ /dev/null @@ -1,1385 +0,0 @@ -/* - string_format.h -- implementation of string.format(). - - It uses the Objects/stringlib conventions, so that it can be - compiled for both unicode and string objects. -*/ - - -/* Defines for Python 2.6 compatibility */ -#if PY_VERSION_HEX < 0x03000000 -#define PyLong_FromSsize_t _PyLong_FromSsize_t -#endif - -/* Defines for more efficiently reallocating the string buffer */ -#define INITIAL_SIZE_INCREMENT 100 -#define SIZE_MULTIPLIER 2 -#define MAX_SIZE_INCREMENT 3200 - - -/************************************************************************/ -/*********** Global data structures and forward declarations *********/ -/************************************************************************/ - -/* - A SubString consists of the characters between two string or - unicode pointers. -*/ -typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; -} SubString; - - -typedef enum { - ANS_INIT, - ANS_AUTO, - ANS_MANUAL -} AutoNumberState; /* Keep track if we're auto-numbering fields */ - -/* Keeps track of our auto-numbering state, and which number field we're on */ -typedef struct { - AutoNumberState an_state; - int an_field_number; -} AutoNumber; - - -/* forward declaration for recursion */ -static PyObject * -build_string(SubString *input, PyObject *args, PyObject *kwargs, - int recursion_depth, AutoNumber *auto_number); - - - -/************************************************************************/ -/************************** Utility functions ************************/ -/************************************************************************/ - -static void -AutoNumber_Init(AutoNumber *auto_number) -{ - auto_number->an_state = ANS_INIT; - auto_number->an_field_number = 0; -} - -/* fill in a SubString from a pointer and length */ -Py_LOCAL_INLINE(void) -SubString_init(SubString *str, STRINGLIB_CHAR *p, Py_ssize_t len) -{ - str->ptr = p; - if (p == NULL) - str->end = NULL; - else - str->end = str->ptr + len; -} - -/* return a new string. if str->ptr is NULL, return None */ -Py_LOCAL_INLINE(PyObject *) -SubString_new_object(SubString *str) -{ - if (str->ptr == NULL) { - Py_INCREF(Py_None); - return Py_None; - } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); -} - -/* return a new string. if str->ptr is NULL, return None */ -Py_LOCAL_INLINE(PyObject *) -SubString_new_object_or_empty(SubString *str) -{ - if (str->ptr == NULL) { - return STRINGLIB_NEW(NULL, 0); - } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); -} - -/* Return 1 if an error has been detected switching between automatic - field numbering and manual field specification, else return 0. Set - ValueError on error. */ -static int -autonumber_state_error(AutoNumberState state, int field_name_is_empty) -{ - if (state == ANS_MANUAL) { - if (field_name_is_empty) { - PyErr_SetString(PyExc_ValueError, "cannot switch from " - "manual field specification to " - "automatic field numbering"); - return 1; - } - } - else { - if (!field_name_is_empty) { - PyErr_SetString(PyExc_ValueError, "cannot switch from " - "automatic field numbering to " - "manual field specification"); - return 1; - } - } - return 0; -} - - -/************************************************************************/ -/*********** Output string management functions ****************/ -/************************************************************************/ - -typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; - PyObject *obj; - Py_ssize_t size_increment; -} OutputString; - -/* initialize an OutputString object, reserving size characters */ -static int -output_initialize(OutputString *output, Py_ssize_t size) -{ - output->obj = STRINGLIB_NEW(NULL, size); - if (output->obj == NULL) - return 0; - - output->ptr = STRINGLIB_STR(output->obj); - output->end = STRINGLIB_LEN(output->obj) + output->ptr; - output->size_increment = INITIAL_SIZE_INCREMENT; - - return 1; -} - -/* - output_extend reallocates the output string buffer. - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ - -static int -output_extend(OutputString *output, Py_ssize_t count) -{ - STRINGLIB_CHAR *startptr = STRINGLIB_STR(output->obj); - Py_ssize_t curlen = output->ptr - startptr; - Py_ssize_t maxlen = curlen + count + output->size_increment; - - if (STRINGLIB_RESIZE(&output->obj, maxlen) < 0) - return 0; - startptr = STRINGLIB_STR(output->obj); - output->ptr = startptr + curlen; - output->end = startptr + maxlen; - if (output->size_increment < MAX_SIZE_INCREMENT) - output->size_increment *= SIZE_MULTIPLIER; - return 1; -} - -/* - output_data dumps characters into our output string - buffer. - - In some cases, it has to reallocate the string. - - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ -static int -output_data(OutputString *output, const STRINGLIB_CHAR *s, Py_ssize_t count) -{ - if ((count > output->end - output->ptr) && !output_extend(output, count)) - return 0; - memcpy(output->ptr, s, count * sizeof(STRINGLIB_CHAR)); - output->ptr += count; - return 1; -} - -/************************************************************************/ -/*********** Format string parsing -- integers and identifiers *********/ -/************************************************************************/ - -static Py_ssize_t -get_integer(const SubString *str) -{ - Py_ssize_t accumulator = 0; - Py_ssize_t digitval; - Py_ssize_t oldaccumulator; - STRINGLIB_CHAR *p; - - /* empty string is an error */ - if (str->ptr >= str->end) - return -1; - - for (p = str->ptr; p < str->end; p++) { - digitval = STRINGLIB_TODECIMAL(*p); - if (digitval < 0) - return -1; - /* - Detect possible overflow before it happens: - - accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if - accumulator > (PY_SSIZE_T_MAX - digitval) / 10. - */ - if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { - PyErr_Format(PyExc_ValueError, - "Too many decimal digits in format string"); - return -1; - } - accumulator = accumulator * 10 + digitval; - } - return accumulator; -} - -/************************************************************************/ -/******** Functions to get field objects and specification strings ******/ -/************************************************************************/ - -/* do the equivalent of obj.name */ -static PyObject * -getattr(PyObject *obj, SubString *name) -{ - PyObject *newobj; - PyObject *str = SubString_new_object(name); - if (str == NULL) - return NULL; - newobj = PyObject_GetAttr(obj, str); - Py_DECREF(str); - return newobj; -} - -/* do the equivalent of obj[idx], where obj is a sequence */ -static PyObject * -getitem_sequence(PyObject *obj, Py_ssize_t idx) -{ - return PySequence_GetItem(obj, idx); -} - -/* do the equivalent of obj[idx], where obj is not a sequence */ -static PyObject * -getitem_idx(PyObject *obj, Py_ssize_t idx) -{ - PyObject *newobj; - PyObject *idx_obj = PyLong_FromSsize_t(idx); - if (idx_obj == NULL) - return NULL; - newobj = PyObject_GetItem(obj, idx_obj); - Py_DECREF(idx_obj); - return newobj; -} - -/* do the equivalent of obj[name] */ -static PyObject * -getitem_str(PyObject *obj, SubString *name) -{ - PyObject *newobj; - PyObject *str = SubString_new_object(name); - if (str == NULL) - return NULL; - newobj = PyObject_GetItem(obj, str); - Py_DECREF(str); - return newobj; -} - -typedef struct { - /* the entire string we're parsing. we assume that someone else - is managing its lifetime, and that it will exist for the - lifetime of the iterator. can be empty */ - SubString str; - - /* pointer to where we are inside field_name */ - STRINGLIB_CHAR *ptr; -} FieldNameIterator; - - -static int -FieldNameIterator_init(FieldNameIterator *self, STRINGLIB_CHAR *ptr, - Py_ssize_t len) -{ - SubString_init(&self->str, ptr, len); - self->ptr = self->str.ptr; - return 1; -} - -static int -_FieldNameIterator_attr(FieldNameIterator *self, SubString *name) -{ - STRINGLIB_CHAR c; - - name->ptr = self->ptr; - - /* return everything until '.' or '[' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { - case '[': - case '.': - /* backup so that we this character will be seen next time */ - self->ptr--; - break; - default: - continue; - } - break; - } - /* end of string is okay */ - name->end = self->ptr; - return 1; -} - -static int -_FieldNameIterator_item(FieldNameIterator *self, SubString *name) -{ - int bracket_seen = 0; - STRINGLIB_CHAR c; - - name->ptr = self->ptr; - - /* return everything until ']' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { - case ']': - bracket_seen = 1; - break; - default: - continue; - } - break; - } - /* make sure we ended with a ']' */ - if (!bracket_seen) { - PyErr_SetString(PyExc_ValueError, "Missing ']' in format string"); - return 0; - } - - /* end of string is okay */ - /* don't include the ']' */ - name->end = self->ptr-1; - return 1; -} - -/* returns 0 on error, 1 on non-error termination, and 2 if it returns a value */ -static int -FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, - Py_ssize_t *name_idx, SubString *name) -{ - /* check at end of input */ - if (self->ptr >= self->str.end) - return 1; - - switch (*self->ptr++) { - case '.': - *is_attribute = 1; - if (_FieldNameIterator_attr(self, name) == 0) - return 0; - *name_idx = -1; - break; - case '[': - *is_attribute = 0; - if (_FieldNameIterator_item(self, name) == 0) - return 0; - *name_idx = get_integer(name); - if (*name_idx == -1 && PyErr_Occurred()) - return 0; - break; - default: - /* Invalid character follows ']' */ - PyErr_SetString(PyExc_ValueError, "Only '.' or '[' may " - "follow ']' in format field specifier"); - return 0; - } - - /* empty string is an error */ - if (name->ptr == name->end) { - PyErr_SetString(PyExc_ValueError, "Empty attribute in format string"); - return 0; - } - - return 2; -} - - -/* input: field_name - output: 'first' points to the part before the first '[' or '.' - 'first_idx' is -1 if 'first' is not an integer, otherwise - it's the value of first converted to an integer - 'rest' is an iterator to return the rest -*/ -static int -field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first, - Py_ssize_t *first_idx, FieldNameIterator *rest, - AutoNumber *auto_number) -{ - STRINGLIB_CHAR c; - STRINGLIB_CHAR *p = ptr; - STRINGLIB_CHAR *end = ptr + len; - int field_name_is_empty; - int using_numeric_index; - - /* find the part up until the first '.' or '[' */ - while (p < end) { - switch (c = *p++) { - case '[': - case '.': - /* backup so that we this character is available to the - "rest" iterator */ - p--; - break; - default: - continue; - } - break; - } - - /* set up the return values */ - SubString_init(first, ptr, p - ptr); - FieldNameIterator_init(rest, p, end - p); - - /* see if "first" is an integer, in which case it's used as an index */ - *first_idx = get_integer(first); - if (*first_idx == -1 && PyErr_Occurred()) - return 0; - - field_name_is_empty = first->ptr >= first->end; - - /* If the field name is omitted or if we have a numeric index - specified, then we're doing numeric indexing into args. */ - using_numeric_index = field_name_is_empty || *first_idx != -1; - - /* We always get here exactly one time for each field we're - processing. And we get here in field order (counting by left - braces). So this is the perfect place to handle automatic field - numbering if the field name is omitted. */ - - /* Check if we need to do the auto-numbering. It's not needed if - we're called from string.Format routines, because it's handled - in that class by itself. */ - if (auto_number) { - /* Initialize our auto numbering state if this is the first - time we're either auto-numbering or manually numbering. */ - if (auto_number->an_state == ANS_INIT && using_numeric_index) - auto_number->an_state = field_name_is_empty ? - ANS_AUTO : ANS_MANUAL; - - /* Make sure our state is consistent with what we're doing - this time through. Only check if we're using a numeric - index. */ - if (using_numeric_index) - if (autonumber_state_error(auto_number->an_state, - field_name_is_empty)) - return 0; - /* Zero length field means we want to do auto-numbering of the - fields. */ - if (field_name_is_empty) - *first_idx = (auto_number->an_field_number)++; - } - - return 1; -} - - -/* - get_field_object returns the object inside {}, before the - format_spec. It handles getindex and getattr lookups and consumes - the entire input string. -*/ -static PyObject * -get_field_object(SubString *input, PyObject *args, PyObject *kwargs, - AutoNumber *auto_number) -{ - PyObject *obj = NULL; - int ok; - int is_attribute; - SubString name; - SubString first; - Py_ssize_t index; - FieldNameIterator rest; - - if (!field_name_split(input->ptr, input->end - input->ptr, &first, - &index, &rest, auto_number)) { - goto error; - } - - if (index == -1) { - /* look up in kwargs */ - PyObject *key = SubString_new_object(&first); - if (key == NULL) - goto error; - - /* Use PyObject_GetItem instead of PyDict_GetItem because this - code is no longer just used with kwargs. It might be passed - a non-dict when called through format_map. */ - if ((kwargs == NULL) || (obj = PyObject_GetItem(kwargs, key)) == NULL) { - PyErr_SetObject(PyExc_KeyError, key); - Py_DECREF(key); - goto error; - } - Py_DECREF(key); - } - else { - /* If args is NULL, we have a format string with a positional field - with only kwargs to retrieve it from. This can only happen when - used with format_map(), where positional arguments are not - allowed. */ - if (args == NULL) { - PyErr_SetString(PyExc_ValueError, "Format string contains " - "positional fields"); - goto error; - } - - /* look up in args */ - obj = PySequence_GetItem(args, index); - if (obj == NULL) - goto error; - } - - /* iterate over the rest of the field_name */ - while ((ok = FieldNameIterator_next(&rest, &is_attribute, &index, - &name)) == 2) { - PyObject *tmp; - - if (is_attribute) - /* getattr lookup "." */ - tmp = getattr(obj, &name); - else - /* getitem lookup "[]" */ - if (index == -1) - tmp = getitem_str(obj, &name); - else - if (PySequence_Check(obj)) - tmp = getitem_sequence(obj, index); - else - /* not a sequence */ - tmp = getitem_idx(obj, index); - if (tmp == NULL) - goto error; - - /* assign to obj */ - Py_DECREF(obj); - obj = tmp; - } - /* end of iterator, this is the non-error case */ - if (ok == 1) - return obj; -error: - Py_XDECREF(obj); - return NULL; -} - -/************************************************************************/ -/***************** Field rendering functions **************************/ -/************************************************************************/ - -/* - render_field() is the main function in this section. It takes the - field object and field specification string generated by - get_field_and_spec, and renders the field into the output string. - - render_field calls fieldobj.__format__(format_spec) method, and - appends to the output. -*/ -static int -render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) -{ - int ok = 0; - PyObject *result = NULL; - PyObject *format_spec_object = NULL; - PyObject *(*formatter)(PyObject *, STRINGLIB_CHAR *, Py_ssize_t) = NULL; - STRINGLIB_CHAR* format_spec_start = format_spec->ptr ? - format_spec->ptr : NULL; - Py_ssize_t format_spec_len = format_spec->ptr ? - format_spec->end - format_spec->ptr : 0; - - /* If we know the type exactly, skip the lookup of __format__ and just - call the formatter directly. */ - if (PyUnicode_CheckExact(fieldobj)) - formatter = _PyUnicode_FormatAdvanced; - else if (PyLong_CheckExact(fieldobj)) - formatter =_PyLong_FormatAdvanced; - else if (PyFloat_CheckExact(fieldobj)) - formatter = _PyFloat_FormatAdvanced; - - /* XXX: for 2.6, convert format_spec to the appropriate type - (unicode, str) */ - - if (formatter) { - /* we know exactly which formatter will be called when __format__ is - looked up, so call it directly, instead. */ - result = formatter(fieldobj, format_spec_start, format_spec_len); - } - else { - /* We need to create an object out of the pointers we have, because - __format__ takes a string/unicode object for format_spec. */ - format_spec_object = STRINGLIB_NEW(format_spec_start, - format_spec_len); - if (format_spec_object == NULL) - goto done; - - result = PyObject_Format(fieldobj, format_spec_object); - } - if (result == NULL) - goto done; - -#if PY_VERSION_HEX >= 0x03000000 - assert(PyUnicode_Check(result)); -#else - assert(PyBytes_Check(result) || PyUnicode_Check(result)); - - /* Convert result to our type. We could be str, and result could - be unicode */ - { - PyObject *tmp = STRINGLIB_TOSTR(result); - if (tmp == NULL) - goto done; - Py_DECREF(result); - result = tmp; - } -#endif - - ok = output_data(output, - STRINGLIB_STR(result), STRINGLIB_LEN(result)); -done: - Py_XDECREF(format_spec_object); - Py_XDECREF(result); - return ok; -} - -static int -parse_field(SubString *str, SubString *field_name, SubString *format_spec, - STRINGLIB_CHAR *conversion) -{ - /* Note this function works if the field name is zero length, - which is good. Zero length field names are handled later, in - field_name_split. */ - - STRINGLIB_CHAR c = 0; - - /* initialize these, as they may be empty */ - *conversion = '\0'; - SubString_init(format_spec, NULL, 0); - - /* Search for the field name. it's terminated by the end of - the string, or a ':' or '!' */ - field_name->ptr = str->ptr; - while (str->ptr < str->end) { - switch (c = *(str->ptr++)) { - case ':': - case '!': - break; - default: - continue; - } - break; - } - - if (c == '!' || c == ':') { - /* we have a format specifier and/or a conversion */ - /* don't include the last character */ - field_name->end = str->ptr-1; - - /* the format specifier is the rest of the string */ - format_spec->ptr = str->ptr; - format_spec->end = str->end; - - /* see if there's a conversion specifier */ - if (c == '!') { - /* there must be another character present */ - if (format_spec->ptr >= format_spec->end) { - PyErr_SetString(PyExc_ValueError, - "end of format while looking for conversion " - "specifier"); - return 0; - } - *conversion = *(format_spec->ptr++); - - /* if there is another character, it must be a colon */ - if (format_spec->ptr < format_spec->end) { - c = *(format_spec->ptr++); - if (c != ':') { - PyErr_SetString(PyExc_ValueError, - "expected ':' after format specifier"); - return 0; - } - } - } - } - else - /* end of string, there's no format_spec or conversion */ - field_name->end = str->ptr; - - return 1; -} - -/************************************************************************/ -/******* Output string allocation and escape-to-markup processing ******/ -/************************************************************************/ - -/* MarkupIterator breaks the string into pieces of either literal - text, or things inside {} that need to be marked up. it is - designed to make it easy to wrap a Python iterator around it, for - use with the Formatter class */ - -typedef struct { - SubString str; -} MarkupIterator; - -static int -MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len) -{ - SubString_init(&self->str, ptr, len); - return 1; -} - -/* returns 0 on error, 1 on non-error termination, and 2 if it got a - string (or something to be expanded) */ -static int -MarkupIterator_next(MarkupIterator *self, SubString *literal, - int *field_present, SubString *field_name, - SubString *format_spec, STRINGLIB_CHAR *conversion, - int *format_spec_needs_expanding) -{ - int at_end; - STRINGLIB_CHAR c = 0; - STRINGLIB_CHAR *start; - int count; - Py_ssize_t len; - int markup_follows = 0; - - /* initialize all of the output variables */ - SubString_init(literal, NULL, 0); - SubString_init(field_name, NULL, 0); - SubString_init(format_spec, NULL, 0); - *conversion = '\0'; - *format_spec_needs_expanding = 0; - *field_present = 0; - - /* No more input, end of iterator. This is the normal exit - path. */ - if (self->str.ptr >= self->str.end) - return 1; - - start = self->str.ptr; - - /* First read any literal text. Read until the end of string, an - escaped '{' or '}', or an unescaped '{'. In order to never - allocate memory and so I can just pass pointers around, if - there's an escaped '{' or '}' then we'll return the literal - including the brace, but no format object. The next time - through, we'll return the rest of the literal, skipping past - the second consecutive brace. */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { - case '{': - case '}': - markup_follows = 1; - break; - default: - continue; - } - break; - } - - at_end = self->str.ptr >= self->str.end; - len = self->str.ptr - start; - - if ((c == '}') && (at_end || (c != *self->str.ptr))) { - PyErr_SetString(PyExc_ValueError, "Single '}' encountered " - "in format string"); - return 0; - } - if (at_end && c == '{') { - PyErr_SetString(PyExc_ValueError, "Single '{' encountered " - "in format string"); - return 0; - } - if (!at_end) { - if (c == *self->str.ptr) { - /* escaped } or {, skip it in the input. there is no - markup object following us, just this literal text */ - self->str.ptr++; - markup_follows = 0; - } - else - len--; - } - - /* record the literal text */ - literal->ptr = start; - literal->end = start + len; - - if (!markup_follows) - return 2; - - /* this is markup, find the end of the string by counting nested - braces. note that this prohibits escaped braces, so that - format_specs cannot have braces in them. */ - *field_present = 1; - count = 1; - - start = self->str.ptr; - - /* we know we can't have a zero length string, so don't worry - about that case */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { - case '{': - /* the format spec needs to be recursively expanded. - this is an optimization, and not strictly needed */ - *format_spec_needs_expanding = 1; - count++; - break; - case '}': - count--; - if (count <= 0) { - /* we're done. parse and get out */ - SubString s; - - SubString_init(&s, start, self->str.ptr - 1 - start); - if (parse_field(&s, field_name, format_spec, conversion) == 0) - return 0; - - /* success */ - return 2; - } - break; - } - } - - /* end of string while searching for matching '}' */ - PyErr_SetString(PyExc_ValueError, "unmatched '{' in format"); - return 0; -} - - -/* do the !r or !s conversion on obj */ -static PyObject * -do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) -{ - /* XXX in pre-3.0, do we need to convert this to unicode, since it - might have returned a string? */ - switch (conversion) { - case 'r': - return PyObject_Repr(obj); - case 's': - return STRINGLIB_TOSTR(obj); -#if PY_VERSION_HEX >= 0x03000000 - case 'a': - return STRINGLIB_TOASCII(obj); -#endif - default: - if (conversion > 32 && conversion < 127) { - /* It's the ASCII subrange; casting to char is safe - (assuming the execution character set is an ASCII - superset). */ - PyErr_Format(PyExc_ValueError, - "Unknown conversion specifier %c", - (char)conversion); - } else - PyErr_Format(PyExc_ValueError, - "Unknown conversion specifier \\x%x", - (unsigned int)conversion); - return NULL; - } -} - -/* given: - - {field_name!conversion:format_spec} - - compute the result and write it to output. - format_spec_needs_expanding is an optimization. if it's false, - just output the string directly, otherwise recursively expand the - format_spec string. - - field_name is allowed to be zero length, in which case we - are doing auto field numbering. -*/ - -static int -output_markup(SubString *field_name, SubString *format_spec, - int format_spec_needs_expanding, STRINGLIB_CHAR conversion, - OutputString *output, PyObject *args, PyObject *kwargs, - int recursion_depth, AutoNumber *auto_number) -{ - PyObject *tmp = NULL; - PyObject *fieldobj = NULL; - SubString expanded_format_spec; - SubString *actual_format_spec; - int result = 0; - - /* convert field_name to an object */ - fieldobj = get_field_object(field_name, args, kwargs, auto_number); - if (fieldobj == NULL) - goto done; - - if (conversion != '\0') { - tmp = do_conversion(fieldobj, conversion); - if (tmp == NULL) - goto done; - - /* do the assignment, transferring ownership: fieldobj = tmp */ - Py_DECREF(fieldobj); - fieldobj = tmp; - tmp = NULL; - } - - /* if needed, recurively compute the format_spec */ - if (format_spec_needs_expanding) { - tmp = build_string(format_spec, args, kwargs, recursion_depth-1, - auto_number); - if (tmp == NULL) - goto done; - - /* note that in the case we're expanding the format string, - tmp must be kept around until after the call to - render_field. */ - SubString_init(&expanded_format_spec, - STRINGLIB_STR(tmp), STRINGLIB_LEN(tmp)); - actual_format_spec = &expanded_format_spec; - } - else - actual_format_spec = format_spec; - - if (render_field(fieldobj, actual_format_spec, output) == 0) - goto done; - - result = 1; - -done: - Py_XDECREF(fieldobj); - Py_XDECREF(tmp); - - return result; -} - -/* - do_markup is the top-level loop for the format() method. It - searches through the format string for escapes to markup codes, and - calls other functions to move non-markup text to the output, - and to perform the markup to the output. -*/ -static int -do_markup(SubString *input, PyObject *args, PyObject *kwargs, - OutputString *output, int recursion_depth, AutoNumber *auto_number) -{ - MarkupIterator iter; - int format_spec_needs_expanding; - int result; - int field_present; - SubString literal; - SubString field_name; - SubString format_spec; - STRINGLIB_CHAR conversion; - - MarkupIterator_init(&iter, input->ptr, input->end - input->ptr); - while ((result = MarkupIterator_next(&iter, &literal, &field_present, - &field_name, &format_spec, - &conversion, - &format_spec_needs_expanding)) == 2) { - if (!output_data(output, literal.ptr, literal.end - literal.ptr)) - return 0; - if (field_present) - if (!output_markup(&field_name, &format_spec, - format_spec_needs_expanding, conversion, output, - args, kwargs, recursion_depth, auto_number)) - return 0; - } - return result; -} - - -/* - build_string allocates the output string and then - calls do_markup to do the heavy lifting. -*/ -static PyObject * -build_string(SubString *input, PyObject *args, PyObject *kwargs, - int recursion_depth, AutoNumber *auto_number) -{ - OutputString output; - PyObject *result = NULL; - Py_ssize_t count; - - output.obj = NULL; /* needed so cleanup code always works */ - - /* check the recursion level */ - if (recursion_depth <= 0) { - PyErr_SetString(PyExc_ValueError, - "Max string recursion exceeded"); - goto done; - } - - /* initial size is the length of the format string, plus the size - increment. seems like a reasonable default */ - if (!output_initialize(&output, - input->end - input->ptr + - INITIAL_SIZE_INCREMENT)) - goto done; - - if (!do_markup(input, args, kwargs, &output, recursion_depth, - auto_number)) { - goto done; - } - - count = output.ptr - STRINGLIB_STR(output.obj); - if (STRINGLIB_RESIZE(&output.obj, count) < 0) { - goto done; - } - - /* transfer ownership to result */ - result = output.obj; - output.obj = NULL; - -done: - Py_XDECREF(output.obj); - return result; -} - -/************************************************************************/ -/*********** main routine ***********************************************/ -/************************************************************************/ - -/* this is the main entry point */ -static PyObject * -do_string_format(PyObject *self, PyObject *args, PyObject *kwargs) -{ - SubString input; - - /* PEP 3101 says only 2 levels, so that - "{0:{1}}".format('abc', 's') # works - "{0:{1:{2}}}".format('abc', 's', '') # fails - */ - int recursion_depth = 2; - - AutoNumber auto_number; - - AutoNumber_Init(&auto_number); - SubString_init(&input, STRINGLIB_STR(self), STRINGLIB_LEN(self)); - return build_string(&input, args, kwargs, recursion_depth, &auto_number); -} - -static PyObject * -do_string_format_map(PyObject *self, PyObject *obj) -{ - return do_string_format(self, NULL, obj); -} - - -/************************************************************************/ -/*********** formatteriterator ******************************************/ -/************************************************************************/ - -/* This is used to implement string.Formatter.vparse(). It exists so - Formatter can share code with the built in unicode.format() method. - It's really just a wrapper around MarkupIterator that is callable - from Python. */ - -typedef struct { - PyObject_HEAD - - STRINGLIB_OBJECT *str; - - MarkupIterator it_markup; -} formatteriterobject; - -static void -formatteriter_dealloc(formatteriterobject *it) -{ - Py_XDECREF(it->str); - PyObject_FREE(it); -} - -/* returns a tuple: - (literal, field_name, format_spec, conversion) - - literal is any literal text to output. might be zero length - field_name is the string before the ':'. might be None - format_spec is the string after the ':'. mibht be None - conversion is either None, or the string after the '!' -*/ -static PyObject * -formatteriter_next(formatteriterobject *it) -{ - SubString literal; - SubString field_name; - SubString format_spec; - STRINGLIB_CHAR conversion; - int format_spec_needs_expanding; - int field_present; - int result = MarkupIterator_next(&it->it_markup, &literal, &field_present, - &field_name, &format_spec, &conversion, - &format_spec_needs_expanding); - - /* all of the SubString objects point into it->str, so no - memory management needs to be done on them */ - assert(0 <= result && result <= 2); - if (result == 0 || result == 1) - /* if 0, error has already been set, if 1, iterator is empty */ - return NULL; - else { - PyObject *literal_str = NULL; - PyObject *field_name_str = NULL; - PyObject *format_spec_str = NULL; - PyObject *conversion_str = NULL; - PyObject *tuple = NULL; - - literal_str = SubString_new_object(&literal); - if (literal_str == NULL) - goto done; - - field_name_str = SubString_new_object(&field_name); - if (field_name_str == NULL) - goto done; - - /* if field_name is non-zero length, return a string for - format_spec (even if zero length), else return None */ - format_spec_str = (field_present ? - SubString_new_object_or_empty : - SubString_new_object)(&format_spec); - if (format_spec_str == NULL) - goto done; - - /* if the conversion is not specified, return a None, - otherwise create a one length string with the conversion - character */ - if (conversion == '\0') { - conversion_str = Py_None; - Py_INCREF(conversion_str); - } - else - conversion_str = STRINGLIB_NEW(&conversion, 1); - if (conversion_str == NULL) - goto done; - - tuple = PyTuple_Pack(4, literal_str, field_name_str, format_spec_str, - conversion_str); - done: - Py_XDECREF(literal_str); - Py_XDECREF(field_name_str); - Py_XDECREF(format_spec_str); - Py_XDECREF(conversion_str); - return tuple; - } -} - -static PyMethodDef formatteriter_methods[] = { - {NULL, NULL} /* sentinel */ -}; - -static PyTypeObject PyFormatterIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "formatteriterator", /* tp_name */ - sizeof(formatteriterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)formatteriter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)formatteriter_next, /* tp_iternext */ - formatteriter_methods, /* tp_methods */ - 0, -}; - -/* unicode_formatter_parser is used to implement - string.Formatter.vformat. it parses a string and returns tuples - describing the parsed elements. It's a wrapper around - stringlib/string_format.h's MarkupIterator */ -static PyObject * -formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) -{ - formatteriterobject *it; - - if (!PyUnicode_Check(self)) { - PyErr_Format(PyExc_TypeError, "expected str, got %s", Py_TYPE(self)->tp_name); - return NULL; - } - - it = PyObject_New(formatteriterobject, &PyFormatterIter_Type); - if (it == NULL) - return NULL; - - /* take ownership, give the object to the iterator */ - Py_INCREF(self); - it->str = self; - - /* initialize the contained MarkupIterator */ - MarkupIterator_init(&it->it_markup, - STRINGLIB_STR(self), - STRINGLIB_LEN(self)); - - return (PyObject *)it; -} - - -/************************************************************************/ -/*********** fieldnameiterator ******************************************/ -/************************************************************************/ - - -/* This is used to implement string.Formatter.vparse(). It parses the - field name into attribute and item values. It's a Python-callable - wrapper around FieldNameIterator */ - -typedef struct { - PyObject_HEAD - - STRINGLIB_OBJECT *str; - - FieldNameIterator it_field; -} fieldnameiterobject; - -static void -fieldnameiter_dealloc(fieldnameiterobject *it) -{ - Py_XDECREF(it->str); - PyObject_FREE(it); -} - -/* returns a tuple: - (is_attr, value) - is_attr is true if we used attribute syntax (e.g., '.foo') - false if we used index syntax (e.g., '[foo]') - value is an integer or string -*/ -static PyObject * -fieldnameiter_next(fieldnameiterobject *it) -{ - int result; - int is_attr; - Py_ssize_t idx; - SubString name; - - result = FieldNameIterator_next(&it->it_field, &is_attr, - &idx, &name); - if (result == 0 || result == 1) - /* if 0, error has already been set, if 1, iterator is empty */ - return NULL; - else { - PyObject* result = NULL; - PyObject* is_attr_obj = NULL; - PyObject* obj = NULL; - - is_attr_obj = PyBool_FromLong(is_attr); - if (is_attr_obj == NULL) - goto done; - - /* either an integer or a string */ - if (idx != -1) - obj = PyLong_FromSsize_t(idx); - else - obj = SubString_new_object(&name); - if (obj == NULL) - goto done; - - /* return a tuple of values */ - result = PyTuple_Pack(2, is_attr_obj, obj); - - done: - Py_XDECREF(is_attr_obj); - Py_XDECREF(obj); - return result; - } -} - -static PyMethodDef fieldnameiter_methods[] = { - {NULL, NULL} /* sentinel */ -}; - -static PyTypeObject PyFieldNameIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "fieldnameiterator", /* tp_name */ - sizeof(fieldnameiterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)fieldnameiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)fieldnameiter_next, /* tp_iternext */ - fieldnameiter_methods, /* tp_methods */ - 0}; - -/* unicode_formatter_field_name_split is used to implement - string.Formatter.vformat. it takes an PEP 3101 "field name", and - returns a tuple of (first, rest): "first", the part before the - first '.' or '['; and "rest", an iterator for the rest of the field - name. it's a wrapper around stringlib/string_format.h's - field_name_split. The iterator it returns is a - FieldNameIterator */ -static PyObject * -formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) -{ - SubString first; - Py_ssize_t first_idx; - fieldnameiterobject *it; - - PyObject *first_obj = NULL; - PyObject *result = NULL; - - if (!PyUnicode_Check(self)) { - PyErr_Format(PyExc_TypeError, "expected str, got %s", Py_TYPE(self)->tp_name); - return NULL; - } - - it = PyObject_New(fieldnameiterobject, &PyFieldNameIter_Type); - if (it == NULL) - return NULL; - - /* take ownership, give the object to the iterator. this is - just to keep the field_name alive */ - Py_INCREF(self); - it->str = self; - - /* Pass in auto_number = NULL. We'll return an empty string for - first_obj in that case. */ - if (!field_name_split(STRINGLIB_STR(self), - STRINGLIB_LEN(self), - &first, &first_idx, &it->it_field, NULL)) - goto done; - - /* first becomes an integer, if possible; else a string */ - if (first_idx != -1) - first_obj = PyLong_FromSsize_t(first_idx); - else - /* convert "first" into a string object */ - first_obj = SubString_new_object(&first); - if (first_obj == NULL) - goto done; - - /* return a tuple of values */ - result = PyTuple_Pack(2, first_obj, it); - -done: - Py_XDECREF(it); - Py_XDECREF(first_obj); - return result; -} diff --git a/Objects/stringlib/stringdefs.h b/Objects/stringlib/stringdefs.h --- a/Objects/stringlib/stringdefs.h +++ b/Objects/stringlib/stringdefs.h @@ -6,6 +6,8 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 0 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyBytesObject #define STRINGLIB_CHAR char #define STRINGLIB_TYPE_NAME "string" diff --git a/Objects/stringlib/ucs1lib.h b/Objects/stringlib/ucs1lib.h new file mode 100644 --- /dev/null +++ b/Objects/stringlib/ucs1lib.h @@ -0,0 +1,35 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs1lib_fastsearch +#define STRINGLIB(F) ucs1lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_CHAR Py_UCS1 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER +#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER +#define STRINGLIB_FILL Py_UNICODE_FILL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW PyUnicode_FromUCS1 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact +#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping +#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs1_InsertThousandsGrouping +#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs1_InsertThousandsGroupingLocale + + diff --git a/Objects/stringlib/ucs2lib.h b/Objects/stringlib/ucs2lib.h new file mode 100644 --- /dev/null +++ b/Objects/stringlib/ucs2lib.h @@ -0,0 +1,34 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs2lib_fastsearch +#define STRINGLIB(F) ucs2lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_CHAR Py_UCS2 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER +#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER +#define STRINGLIB_FILL Py_UNICODE_FILL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW PyUnicode_FromUCS2 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact +#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping +#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs2_InsertThousandsGrouping +#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs2_InsertThousandsGroupingLocale + diff --git a/Objects/stringlib/ucs4lib.h b/Objects/stringlib/ucs4lib.h new file mode 100644 --- /dev/null +++ b/Objects/stringlib/ucs4lib.h @@ -0,0 +1,34 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs4lib_fastsearch +#define STRINGLIB(F) ucs4lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_CHAR Py_UCS4 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER +#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER +#define STRINGLIB_FILL Py_UNICODE_FILL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW PyUnicode_FromUCS4 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact +#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping +#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs4_InsertThousandsGrouping +#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs4_InsertThousandsGroupingLocale + diff --git a/Objects/stringlib/undef.h b/Objects/stringlib/undef.h new file mode 100644 --- /dev/null +++ b/Objects/stringlib/undef.h @@ -0,0 +1,10 @@ +#undef FASTSEARCH +#undef STRINGLIB +#undef STRINGLIB_CHAR +#undef STRINGLIB_STR +#undef STRINGLIB_LEN +#undef STRINGLIB_NEW +#undef STRINGLIB_RESIZE +#undef _Py_InsertThousandsGrouping +#undef _Py_InsertThousandsGroupingLocale + diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h new file mode 100644 --- /dev/null +++ b/Objects/stringlib/unicode_format.h @@ -0,0 +1,1416 @@ +/* + unicode_format.h -- implementation of str.format(). +*/ + + +/* Defines for more efficiently reallocating the string buffer */ +#define INITIAL_SIZE_INCREMENT 100 +#define SIZE_MULTIPLIER 2 +#define MAX_SIZE_INCREMENT 3200 + + +/************************************************************************/ +/*********** Global data structures and forward declarations *********/ +/************************************************************************/ + +/* + A SubString consists of the characters between two string or + unicode pointers. +*/ +typedef struct { + PyObject *str; /* borrowed reference */ + Py_ssize_t start, end; +} SubString; + + +typedef enum { + ANS_INIT, + ANS_AUTO, + ANS_MANUAL +} AutoNumberState; /* Keep track if we're auto-numbering fields */ + +/* Keeps track of our auto-numbering state, and which number field we're on */ +typedef struct { + AutoNumberState an_state; + int an_field_number; +} AutoNumber; + + +/* forward declaration for recursion */ +static PyObject * +build_string(SubString *input, PyObject *args, PyObject *kwargs, + int recursion_depth, AutoNumber *auto_number); + + + +/************************************************************************/ +/************************** Utility functions ************************/ +/************************************************************************/ + +static void +AutoNumber_Init(AutoNumber *auto_number) +{ + auto_number->an_state = ANS_INIT; + auto_number->an_field_number = 0; +} + +/* fill in a SubString from a pointer and length */ +Py_LOCAL_INLINE(void) +SubString_init(SubString *str, PyObject *s, int start, int end) +{ + str->str = s; + str->start = start; + str->end = end; +} + +/* return a new string. if str->str is NULL, return None */ +Py_LOCAL_INLINE(PyObject *) +SubString_new_object(SubString *str) +{ + if (str->str == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + return PyUnicode_Substring(str->str, str->start, str->end); +} + +/* return a new string. if str->str is NULL, return None */ +Py_LOCAL_INLINE(PyObject *) +SubString_new_object_or_empty(SubString *str) +{ + if (str->str == NULL) { + return PyUnicode_FromUnicode(NULL, 0); + } + return SubString_new_object(str); +} + +/* Return 1 if an error has been detected switching between automatic + field numbering and manual field specification, else return 0. Set + ValueError on error. */ +static int +autonumber_state_error(AutoNumberState state, int field_name_is_empty) +{ + if (state == ANS_MANUAL) { + if (field_name_is_empty) { + PyErr_SetString(PyExc_ValueError, "cannot switch from " + "manual field specification to " + "automatic field numbering"); + return 1; + } + } + else { + if (!field_name_is_empty) { + PyErr_SetString(PyExc_ValueError, "cannot switch from " + "automatic field numbering to " + "manual field specification"); + return 1; + } + } + return 0; +} + + +/************************************************************************/ +/*********** Output string management functions ****************/ +/************************************************************************/ + +typedef struct { + char *data; + Py_UCS4 maxchar; + unsigned int kind; + Py_ssize_t pos, size; + Py_ssize_t size_increment; +} OutputString; + +/* initialize an OutputString object, reserving size characters */ +static int +output_initialize(OutputString *output, Py_ssize_t size) +{ + output->data = PyMem_Malloc(size); + if (output->data == NULL) { + PyErr_NoMemory(); + return 0; + } + + output->maxchar = 127; + output->kind = PyUnicode_1BYTE_KIND; + output->pos = 0; + output->size = size; + output->size_increment = INITIAL_SIZE_INCREMENT; + + return 1; +} + +/* + output_extend reallocates the output string buffer. + It returns a status: 0 for a failed reallocation, + 1 for success. +*/ + +static int +output_extend(OutputString *output, Py_ssize_t count) +{ + Py_ssize_t maxlen = output->size + count + output->size_increment; + + output->data = PyMem_Realloc(output->data, maxlen << (output->kind-1)); + output->size = maxlen; + if (output->data == 0) { + PyErr_NoMemory(); + return 0; + } + if (output->size_increment < MAX_SIZE_INCREMENT) + output->size_increment *= SIZE_MULTIPLIER; + return 1; +} + +static int +output_widen(OutputString *output, Py_UCS4 maxchar) +{ + int kind; + void *data; + Py_ssize_t i; + if (maxchar <= output->maxchar) + return 1; + if (maxchar < 256) { + output->maxchar = 255; + return 1; + } + if (maxchar < 65536) { + output->maxchar = 65535; + kind = 2; + } + else { + output->maxchar = 1<<21; + kind = 3; + } + data = PyMem_Malloc(output->size << (kind-1)); + if (data == 0) + return 0; + for (i = 0; i < output->size; i++) + PyUnicode_WRITE(kind, data, i, + PyUnicode_READ(output->kind, output->data, i)); + PyMem_Free(output->data); + output->data = data; + output->kind = kind; + return 1; +} + +/* + output_data dumps characters into our output string + buffer. + + In some cases, it has to reallocate the string. + + It returns a status: 0 for a failed reallocation, + 1 for success. +*/ +static int +output_data(OutputString *output, PyObject *s, Py_ssize_t start, Py_ssize_t end) +{ + Py_ssize_t i; + int kind; + if ((output->pos + end - start > output->size) && + !output_extend(output, end - start)) + return 0; + kind = PyUnicode_KIND(s); + if (PyUnicode_MAX_CHAR_VALUE(s) > output->maxchar) { + Py_UCS4 maxchar = output->maxchar; + for (i = start; i < end; i++) + if (PyUnicode_READ(kind, PyUnicode_DATA(s), i) > maxchar) + maxchar = PyUnicode_READ(kind, PyUnicode_DATA(s), i); + if (!output_widen(output, maxchar)) + return 0; + } + for (i = start; i < end; i++) + PyUnicode_WRITE(output->kind, output->data, output->pos++, + PyUnicode_READ(kind, PyUnicode_DATA(s), i)); + return 1; +} + +/************************************************************************/ +/*********** Format string parsing -- integers and identifiers *********/ +/************************************************************************/ + +static Py_ssize_t +get_integer(const SubString *str) +{ + Py_ssize_t accumulator = 0; + Py_ssize_t digitval; + Py_ssize_t i; + + /* empty string is an error */ + if (str->start >= str->end) + return -1; + + for (i = str->start; i < str->end; i++) { + digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str->str, i)); + if (digitval < 0) + return -1; + /* + Detect possible overflow before it happens: + + accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if + accumulator > (PY_SSIZE_T_MAX - digitval) / 10. + */ + if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { + PyErr_Format(PyExc_ValueError, + "Too many decimal digits in format string"); + return -1; + } + accumulator = accumulator * 10 + digitval; + } + return accumulator; +} + +/************************************************************************/ +/******** Functions to get field objects and specification strings ******/ +/************************************************************************/ + +/* do the equivalent of obj.name */ +static PyObject * +getattr(PyObject *obj, SubString *name) +{ + PyObject *newobj; + PyObject *str = SubString_new_object(name); + if (str == NULL) + return NULL; + newobj = PyObject_GetAttr(obj, str); + Py_DECREF(str); + return newobj; +} + +/* do the equivalent of obj[idx], where obj is a sequence */ +static PyObject * +getitem_sequence(PyObject *obj, Py_ssize_t idx) +{ + return PySequence_GetItem(obj, idx); +} + +/* do the equivalent of obj[idx], where obj is not a sequence */ +static PyObject * +getitem_idx(PyObject *obj, Py_ssize_t idx) +{ + PyObject *newobj; + PyObject *idx_obj = PyLong_FromSsize_t(idx); + if (idx_obj == NULL) + return NULL; + newobj = PyObject_GetItem(obj, idx_obj); + Py_DECREF(idx_obj); + return newobj; +} + +/* do the equivalent of obj[name] */ +static PyObject * +getitem_str(PyObject *obj, SubString *name) +{ + PyObject *newobj; + PyObject *str = SubString_new_object(name); + if (str == NULL) + return NULL; + newobj = PyObject_GetItem(obj, str); + Py_DECREF(str); + return newobj; +} + +typedef struct { + /* the entire string we're parsing. we assume that someone else + is managing its lifetime, and that it will exist for the + lifetime of the iterator. can be empty */ + SubString str; + + /* index to where we are inside field_name */ + Py_ssize_t index; +} FieldNameIterator; + + +static int +FieldNameIterator_init(FieldNameIterator *self, PyObject *s, + Py_ssize_t start, Py_ssize_t end) +{ + SubString_init(&self->str, s, start, end); + self->index = start; + return 1; +} + +static int +_FieldNameIterator_attr(FieldNameIterator *self, SubString *name) +{ + Py_UCS4 c; + + name->str = self->str.str; + name->start = self->index; + + /* return everything until '.' or '[' */ + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { + case '[': + case '.': + /* backup so that we this character will be seen next time */ + self->index--; + break; + default: + continue; + } + break; + } + /* end of string is okay */ + name->end = self->index; + return 1; +} + +static int +_FieldNameIterator_item(FieldNameIterator *self, SubString *name) +{ + int bracket_seen = 0; + Py_UCS4 c; + + name->str = self->str.str; + name->start = self->index; + + /* return everything until ']' */ + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { + case ']': + bracket_seen = 1; + break; + default: + continue; + } + break; + } + /* make sure we ended with a ']' */ + if (!bracket_seen) { + PyErr_SetString(PyExc_ValueError, "Missing ']' in format string"); + return 0; + } + + /* end of string is okay */ + /* don't include the ']' */ + name->end = self->index-1; + return 1; +} + +/* returns 0 on error, 1 on non-error termination, and 2 if it returns a value */ +static int +FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, + Py_ssize_t *name_idx, SubString *name) +{ + /* check at end of input */ + if (self->index >= self->str.end) + return 1; + + switch (PyUnicode_READ_CHAR(self->str.str, self->index++)) { + case '.': + *is_attribute = 1; + if (_FieldNameIterator_attr(self, name) == 0) + return 0; + *name_idx = -1; + break; + case '[': + *is_attribute = 0; + if (_FieldNameIterator_item(self, name) == 0) + return 0; + *name_idx = get_integer(name); + if (*name_idx == -1 && PyErr_Occurred()) + return 0; + break; + default: + /* Invalid character follows ']' */ + PyErr_SetString(PyExc_ValueError, "Only '.' or '[' may " + "follow ']' in format field specifier"); + return 0; + } + + /* empty string is an error */ + if (name->start == name->end) { + PyErr_SetString(PyExc_ValueError, "Empty attribute in format string"); + return 0; + } + + return 2; +} + + +/* input: field_name + output: 'first' points to the part before the first '[' or '.' + 'first_idx' is -1 if 'first' is not an integer, otherwise + it's the value of first converted to an integer + 'rest' is an iterator to return the rest +*/ +static int +field_name_split(PyObject *str, Py_ssize_t start, Py_ssize_t end, SubString *first, + Py_ssize_t *first_idx, FieldNameIterator *rest, + AutoNumber *auto_number) +{ + Py_UCS4 c; + Py_ssize_t i = start; + int field_name_is_empty; + int using_numeric_index; + + /* find the part up until the first '.' or '[' */ + while (i < end) { + switch (c = PyUnicode_READ_CHAR(str, i++)) { + case '[': + case '.': + /* backup so that we this character is available to the + "rest" iterator */ + i--; + break; + default: + continue; + } + break; + } + + /* set up the return values */ + SubString_init(first, str, start, i); + FieldNameIterator_init(rest, str, i, end); + + /* see if "first" is an integer, in which case it's used as an index */ + *first_idx = get_integer(first); + if (*first_idx == -1 && PyErr_Occurred()) + return 0; + + field_name_is_empty = first->start >= first->end; + + /* If the field name is omitted or if we have a numeric index + specified, then we're doing numeric indexing into args. */ + using_numeric_index = field_name_is_empty || *first_idx != -1; + + /* We always get here exactly one time for each field we're + processing. And we get here in field order (counting by left + braces). So this is the perfect place to handle automatic field + numbering if the field name is omitted. */ + + /* Check if we need to do the auto-numbering. It's not needed if + we're called from string.Format routines, because it's handled + in that class by itself. */ + if (auto_number) { + /* Initialize our auto numbering state if this is the first + time we're either auto-numbering or manually numbering. */ + if (auto_number->an_state == ANS_INIT && using_numeric_index) + auto_number->an_state = field_name_is_empty ? + ANS_AUTO : ANS_MANUAL; + + /* Make sure our state is consistent with what we're doing + this time through. Only check if we're using a numeric + index. */ + if (using_numeric_index) + if (autonumber_state_error(auto_number->an_state, + field_name_is_empty)) + return 0; + /* Zero length field means we want to do auto-numbering of the + fields. */ + if (field_name_is_empty) + *first_idx = (auto_number->an_field_number)++; + } + + return 1; +} + + +/* + get_field_object returns the object inside {}, before the + format_spec. It handles getindex and getattr lookups and consumes + the entire input string. +*/ +static PyObject * +get_field_object(SubString *input, PyObject *args, PyObject *kwargs, + AutoNumber *auto_number) +{ + PyObject *obj = NULL; + int ok; + int is_attribute; + SubString name; + SubString first; + Py_ssize_t index; + FieldNameIterator rest; + + if (!field_name_split(input->str, input->start, input->end, &first, + &index, &rest, auto_number)) { + goto error; + } + + if (index == -1) { + /* look up in kwargs */ + PyObject *key = SubString_new_object(&first); + if (key == NULL) + goto error; + + /* Use PyObject_GetItem instead of PyDict_GetItem because this + code is no longer just used with kwargs. It might be passed + a non-dict when called through format_map. */ + if ((kwargs == NULL) || (obj = PyObject_GetItem(kwargs, key)) == NULL) { + PyErr_SetObject(PyExc_KeyError, key); + Py_DECREF(key); + goto error; + } + Py_DECREF(key); + } + else { + /* If args is NULL, we have a format string with a positional field + with only kwargs to retrieve it from. This can only happen when + used with format_map(), where positional arguments are not + allowed. */ + if (args == NULL) { + PyErr_SetString(PyExc_ValueError, "Format string contains " + "positional fields"); + goto error; + } + + /* look up in args */ + obj = PySequence_GetItem(args, index); + if (obj == NULL) + goto error; + } + + /* iterate over the rest of the field_name */ + while ((ok = FieldNameIterator_next(&rest, &is_attribute, &index, + &name)) == 2) { + PyObject *tmp; + + if (is_attribute) + /* getattr lookup "." */ + tmp = getattr(obj, &name); + else + /* getitem lookup "[]" */ + if (index == -1) + tmp = getitem_str(obj, &name); + else + if (PySequence_Check(obj)) + tmp = getitem_sequence(obj, index); + else + /* not a sequence */ + tmp = getitem_idx(obj, index); + if (tmp == NULL) + goto error; + + /* assign to obj */ + Py_DECREF(obj); + obj = tmp; + } + /* end of iterator, this is the non-error case */ + if (ok == 1) + return obj; +error: + Py_XDECREF(obj); + return NULL; +} + +/************************************************************************/ +/***************** Field rendering functions **************************/ +/************************************************************************/ + +/* + render_field() is the main function in this section. It takes the + field object and field specification string generated by + get_field_and_spec, and renders the field into the output string. + + render_field calls fieldobj.__format__(format_spec) method, and + appends to the output. +*/ +static int +render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) +{ + int ok = 0; + PyObject *result = NULL; + PyObject *format_spec_object = NULL; + PyObject *(*formatter)(PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; + + /* If we know the type exactly, skip the lookup of __format__ and just + call the formatter directly. */ + if (PyUnicode_CheckExact(fieldobj)) + formatter = _PyUnicode_FormatAdvanced; + else if (PyLong_CheckExact(fieldobj)) + formatter =_PyLong_FormatAdvanced; + else if (PyFloat_CheckExact(fieldobj)) + formatter = _PyFloat_FormatAdvanced; + + /* XXX: for 2.6, convert format_spec to the appropriate type + (unicode, str) */ + + if (formatter) { + /* we know exactly which formatter will be called when __format__ is + looked up, so call it directly, instead. */ + result = formatter(fieldobj, format_spec->str, + format_spec->start, format_spec->end); + } + else { + /* We need to create an object out of the pointers we have, because + __format__ takes a string/unicode object for format_spec. */ + if (format_spec->str) + format_spec_object = PyUnicode_Substring(format_spec->str, + format_spec->start, + format_spec->end); + else + format_spec_object = PyUnicode_New(0, 0); + if (format_spec_object == NULL) + goto done; + + result = PyObject_Format(fieldobj, format_spec_object); + } + if (result == NULL || PyUnicode_READY(result) == -1) + goto done; + + assert(PyUnicode_Check(result)); + ok = output_data(output, result, 0, PyUnicode_GET_LENGTH(result)); +done: + Py_XDECREF(format_spec_object); + Py_XDECREF(result); + return ok; +} + +static int +parse_field(SubString *str, SubString *field_name, SubString *format_spec, + Py_UCS4 *conversion) +{ + /* Note this function works if the field name is zero length, + which is good. Zero length field names are handled later, in + field_name_split. */ + + Py_UCS4 c = 0; + + /* initialize these, as they may be empty */ + *conversion = '\0'; + SubString_init(format_spec, NULL, 0, 0); + + /* Search for the field name. it's terminated by the end of + the string, or a ':' or '!' */ + field_name->str = str->str; + field_name->start = str->start; + while (str->start < str->end) { + switch ((c = PyUnicode_READ_CHAR(str->str, str->start++))) { + case ':': + case '!': + break; + default: + continue; + } + break; + } + + if (c == '!' || c == ':') { + /* we have a format specifier and/or a conversion */ + /* don't include the last character */ + field_name->end = str->start-1; + + /* the format specifier is the rest of the string */ + format_spec->str = str->str; + format_spec->start = str->start; + format_spec->end = str->end; + + /* see if there's a conversion specifier */ + if (c == '!') { + /* there must be another character present */ + if (format_spec->start >= format_spec->end) { + PyErr_SetString(PyExc_ValueError, + "end of format while looking for conversion " + "specifier"); + return 0; + } + *conversion = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); + + /* if there is another character, it must be a colon */ + if (format_spec->start < format_spec->end) { + c = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); + if (c != ':') { + PyErr_SetString(PyExc_ValueError, + "expected ':' after format specifier"); + return 0; + } + } + } + } + else + /* end of string, there's no format_spec or conversion */ + field_name->end = str->start; + + return 1; +} + +/************************************************************************/ +/******* Output string allocation and escape-to-markup processing ******/ +/************************************************************************/ + +/* MarkupIterator breaks the string into pieces of either literal + text, or things inside {} that need to be marked up. it is + designed to make it easy to wrap a Python iterator around it, for + use with the Formatter class */ + +typedef struct { + SubString str; +} MarkupIterator; + +static int +MarkupIterator_init(MarkupIterator *self, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + SubString_init(&self->str, str, start, end); + return 1; +} + +/* returns 0 on error, 1 on non-error termination, and 2 if it got a + string (or something to be expanded) */ +static int +MarkupIterator_next(MarkupIterator *self, SubString *literal, + int *field_present, SubString *field_name, + SubString *format_spec, Py_UCS4 *conversion, + int *format_spec_needs_expanding) +{ + int at_end; + Py_UCS4 c = 0; + Py_ssize_t start; + int count; + Py_ssize_t len; + int markup_follows = 0; + + /* initialize all of the output variables */ + SubString_init(literal, NULL, 0, 0); + SubString_init(field_name, NULL, 0, 0); + SubString_init(format_spec, NULL, 0, 0); + *conversion = '\0'; + *format_spec_needs_expanding = 0; + *field_present = 0; + + /* No more input, end of iterator. This is the normal exit + path. */ + if (self->str.start >= self->str.end) + return 1; + + start = self->str.start; + + /* First read any literal text. Read until the end of string, an + escaped '{' or '}', or an unescaped '{'. In order to never + allocate memory and so I can just pass pointers around, if + there's an escaped '{' or '}' then we'll return the literal + including the brace, but no format object. The next time + through, we'll return the rest of the literal, skipping past + the second consecutive brace. */ + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { + case '{': + case '}': + markup_follows = 1; + break; + default: + continue; + } + break; + } + + at_end = self->str.start >= self->str.end; + len = self->str.start - start; + + if ((c == '}') && (at_end || + (c != PyUnicode_READ_CHAR(self->str.str, + self->str.start)))) { + PyErr_SetString(PyExc_ValueError, "Single '}' encountered " + "in format string"); + return 0; + } + if (at_end && c == '{') { + PyErr_SetString(PyExc_ValueError, "Single '{' encountered " + "in format string"); + return 0; + } + if (!at_end) { + if (c == PyUnicode_READ_CHAR(self->str.str, self->str.start)) { + /* escaped } or {, skip it in the input. there is no + markup object following us, just this literal text */ + self->str.start++; + markup_follows = 0; + } + else + len--; + } + + /* record the literal text */ + literal->str = self->str.str; + literal->start = start; + literal->end = start + len; + + if (!markup_follows) + return 2; + + /* this is markup, find the end of the string by counting nested + braces. note that this prohibits escaped braces, so that + format_specs cannot have braces in them. */ + *field_present = 1; + count = 1; + + start = self->str.start; + + /* we know we can't have a zero length string, so don't worry + about that case */ + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { + case '{': + /* the format spec needs to be recursively expanded. + this is an optimization, and not strictly needed */ + *format_spec_needs_expanding = 1; + count++; + break; + case '}': + count--; + if (count <= 0) { + /* we're done. parse and get out */ + SubString s; + + SubString_init(&s, self->str.str, start, self->str.start - 1); + if (parse_field(&s, field_name, format_spec, conversion) == 0) + return 0; + + /* success */ + return 2; + } + break; + } + } + + /* end of string while searching for matching '}' */ + PyErr_SetString(PyExc_ValueError, "unmatched '{' in format"); + return 0; +} + + +/* do the !r or !s conversion on obj */ +static PyObject * +do_conversion(PyObject *obj, Py_UCS4 conversion) +{ + /* XXX in pre-3.0, do we need to convert this to unicode, since it + might have returned a string? */ + switch (conversion) { + case 'r': + return PyObject_Repr(obj); + case 's': + return PyObject_Str(obj); + case 'a': + return PyObject_ASCII(obj); + default: + if (conversion > 32 && conversion < 127) { + /* It's the ASCII subrange; casting to char is safe + (assuming the execution character set is an ASCII + superset). */ + PyErr_Format(PyExc_ValueError, + "Unknown conversion specifier %c", + (char)conversion); + } else + PyErr_Format(PyExc_ValueError, + "Unknown conversion specifier \\x%x", + (unsigned int)conversion); + return NULL; + } +} + +/* given: + + {field_name!conversion:format_spec} + + compute the result and write it to output. + format_spec_needs_expanding is an optimization. if it's false, + just output the string directly, otherwise recursively expand the + format_spec string. + + field_name is allowed to be zero length, in which case we + are doing auto field numbering. +*/ + +static int +output_markup(SubString *field_name, SubString *format_spec, + int format_spec_needs_expanding, Py_UCS4 conversion, + OutputString *output, PyObject *args, PyObject *kwargs, + int recursion_depth, AutoNumber *auto_number) +{ + PyObject *tmp = NULL; + PyObject *fieldobj = NULL; + SubString expanded_format_spec; + SubString *actual_format_spec; + int result = 0; + + /* convert field_name to an object */ + fieldobj = get_field_object(field_name, args, kwargs, auto_number); + if (fieldobj == NULL) + goto done; + + if (conversion != '\0') { + tmp = do_conversion(fieldobj, conversion); + if (tmp == NULL || PyUnicode_READY(tmp) == -1) + goto done; + + /* do the assignment, transferring ownership: fieldobj = tmp */ + Py_DECREF(fieldobj); + fieldobj = tmp; + tmp = NULL; + } + + /* if needed, recurively compute the format_spec */ + if (format_spec_needs_expanding) { + tmp = build_string(format_spec, args, kwargs, recursion_depth-1, + auto_number); + if (tmp == NULL || PyUnicode_READY(tmp) == -1) + goto done; + + /* note that in the case we're expanding the format string, + tmp must be kept around until after the call to + render_field. */ + SubString_init(&expanded_format_spec, tmp, 0, PyUnicode_GET_LENGTH(tmp)); + actual_format_spec = &expanded_format_spec; + } + else + actual_format_spec = format_spec; + + if (render_field(fieldobj, actual_format_spec, output) == 0) + goto done; + + result = 1; + +done: + Py_XDECREF(fieldobj); + Py_XDECREF(tmp); + + return result; +} + +/* + do_markup is the top-level loop for the format() method. It + searches through the format string for escapes to markup codes, and + calls other functions to move non-markup text to the output, + and to perform the markup to the output. +*/ +static int +do_markup(SubString *input, PyObject *args, PyObject *kwargs, + OutputString *output, int recursion_depth, AutoNumber *auto_number) +{ + MarkupIterator iter; + int format_spec_needs_expanding; + int result; + int field_present; + SubString literal; + SubString field_name; + SubString format_spec; + Py_UCS4 conversion; + + MarkupIterator_init(&iter, input->str, input->start, input->end); + while ((result = MarkupIterator_next(&iter, &literal, &field_present, + &field_name, &format_spec, + &conversion, + &format_spec_needs_expanding)) == 2) { + if (!output_data(output, literal.str, literal.start, literal.end)) + return 0; + if (field_present) + if (!output_markup(&field_name, &format_spec, + format_spec_needs_expanding, conversion, output, + args, kwargs, recursion_depth, auto_number)) + return 0; + } + return result; +} + + +/* + build_string allocates the output string and then + calls do_markup to do the heavy lifting. +*/ +static PyObject * +build_string(SubString *input, PyObject *args, PyObject *kwargs, + int recursion_depth, AutoNumber *auto_number) +{ + OutputString output; + PyObject *result = NULL; + + output.data = NULL; /* needed so cleanup code always works */ + + /* check the recursion level */ + if (recursion_depth <= 0) { + PyErr_SetString(PyExc_ValueError, + "Max string recursion exceeded"); + goto done; + } + + /* initial size is the length of the format string, plus the size + increment. seems like a reasonable default */ + if (!output_initialize(&output, + input->end - input->start + + INITIAL_SIZE_INCREMENT)) + goto done; + + if (!do_markup(input, args, kwargs, &output, recursion_depth, + auto_number)) { + goto done; + } + + result = PyUnicode_New(output.pos, output.maxchar); + if (!result) + goto done; + memcpy(PyUnicode_DATA(result), output.data, output.pos << (output.kind-1)); + +done: + if (output.data) + PyMem_Free(output.data); + return result; +} + +/************************************************************************/ +/*********** main routine ***********************************************/ +/************************************************************************/ + +/* this is the main entry point */ +static PyObject * +do_string_format(PyObject *self, PyObject *args, PyObject *kwargs) +{ + SubString input; + + /* PEP 3101 says only 2 levels, so that + "{0:{1}}".format('abc', 's') # works + "{0:{1:{2}}}".format('abc', 's', '') # fails + */ + int recursion_depth = 2; + + AutoNumber auto_number; + + if (PyUnicode_READY(self) == -1) + return NULL; + + AutoNumber_Init(&auto_number); + SubString_init(&input, self, 0, PyUnicode_GET_LENGTH(self)); + return build_string(&input, args, kwargs, recursion_depth, &auto_number); +} + +static PyObject * +do_string_format_map(PyObject *self, PyObject *obj) +{ + return do_string_format(self, NULL, obj); +} + + +/************************************************************************/ +/*********** formatteriterator ******************************************/ +/************************************************************************/ + +/* This is used to implement string.Formatter.vparse(). It exists so + Formatter can share code with the built in unicode.format() method. + It's really just a wrapper around MarkupIterator that is callable + from Python. */ + +typedef struct { + PyObject_HEAD + + PyUnicodeObject *str; + + MarkupIterator it_markup; +} formatteriterobject; + +static void +formatteriter_dealloc(formatteriterobject *it) +{ + Py_XDECREF(it->str); + PyObject_FREE(it); +} + +/* returns a tuple: + (literal, field_name, format_spec, conversion) + + literal is any literal text to output. might be zero length + field_name is the string before the ':'. might be None + format_spec is the string after the ':'. mibht be None + conversion is either None, or the string after the '!' +*/ +static PyObject * +formatteriter_next(formatteriterobject *it) +{ + SubString literal; + SubString field_name; + SubString format_spec; + Py_UCS4 conversion; + int format_spec_needs_expanding; + int field_present; + int result = MarkupIterator_next(&it->it_markup, &literal, &field_present, + &field_name, &format_spec, &conversion, + &format_spec_needs_expanding); + + /* all of the SubString objects point into it->str, so no + memory management needs to be done on them */ + assert(0 <= result && result <= 2); + if (result == 0 || result == 1) + /* if 0, error has already been set, if 1, iterator is empty */ + return NULL; + else { + PyObject *literal_str = NULL; + PyObject *field_name_str = NULL; + PyObject *format_spec_str = NULL; + PyObject *conversion_str = NULL; + PyObject *tuple = NULL; + + literal_str = SubString_new_object(&literal); + if (literal_str == NULL) + goto done; + + field_name_str = SubString_new_object(&field_name); + if (field_name_str == NULL) + goto done; + + /* if field_name is non-zero length, return a string for + format_spec (even if zero length), else return None */ + format_spec_str = (field_present ? + SubString_new_object_or_empty : + SubString_new_object)(&format_spec); + if (format_spec_str == NULL) + goto done; + + /* if the conversion is not specified, return a None, + otherwise create a one length string with the conversion + character */ + if (conversion == '\0') { + conversion_str = Py_None; + Py_INCREF(conversion_str); + } + else + conversion_str = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + &conversion, 1); + if (conversion_str == NULL) + goto done; + + tuple = PyTuple_Pack(4, literal_str, field_name_str, format_spec_str, + conversion_str); + done: + Py_XDECREF(literal_str); + Py_XDECREF(field_name_str); + Py_XDECREF(format_spec_str); + Py_XDECREF(conversion_str); + return tuple; + } +} + +static PyMethodDef formatteriter_methods[] = { + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject PyFormatterIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "formatteriterator", /* tp_name */ + sizeof(formatteriterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)formatteriter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)formatteriter_next, /* tp_iternext */ + formatteriter_methods, /* tp_methods */ + 0, +}; + +/* unicode_formatter_parser is used to implement + string.Formatter.vformat. it parses a string and returns tuples + describing the parsed elements. It's a wrapper around + stringlib/string_format.h's MarkupIterator */ +static PyObject * +formatter_parser(PyObject *ignored, PyUnicodeObject *self) +{ + formatteriterobject *it; + + if (!PyUnicode_Check(self)) { + PyErr_Format(PyExc_TypeError, "expected str, got %s", Py_TYPE(self)->tp_name); + return NULL; + } + + if (PyUnicode_READY(self) == -1) + return NULL; + + it = PyObject_New(formatteriterobject, &PyFormatterIter_Type); + if (it == NULL) + return NULL; + + /* take ownership, give the object to the iterator */ + Py_INCREF(self); + it->str = self; + + /* initialize the contained MarkupIterator */ + MarkupIterator_init(&it->it_markup, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)); + return (PyObject *)it; +} + + +/************************************************************************/ +/*********** fieldnameiterator ******************************************/ +/************************************************************************/ + + +/* This is used to implement string.Formatter.vparse(). It parses the + field name into attribute and item values. It's a Python-callable + wrapper around FieldNameIterator */ + +typedef struct { + PyObject_HEAD + + PyUnicodeObject *str; + + FieldNameIterator it_field; +} fieldnameiterobject; + +static void +fieldnameiter_dealloc(fieldnameiterobject *it) +{ + Py_XDECREF(it->str); + PyObject_FREE(it); +} + +/* returns a tuple: + (is_attr, value) + is_attr is true if we used attribute syntax (e.g., '.foo') + false if we used index syntax (e.g., '[foo]') + value is an integer or string +*/ +static PyObject * +fieldnameiter_next(fieldnameiterobject *it) +{ + int result; + int is_attr; + Py_ssize_t idx; + SubString name; + + result = FieldNameIterator_next(&it->it_field, &is_attr, + &idx, &name); + if (result == 0 || result == 1) + /* if 0, error has already been set, if 1, iterator is empty */ + return NULL; + else { + PyObject* result = NULL; + PyObject* is_attr_obj = NULL; + PyObject* obj = NULL; + + is_attr_obj = PyBool_FromLong(is_attr); + if (is_attr_obj == NULL) + goto done; + + /* either an integer or a string */ + if (idx != -1) + obj = PyLong_FromSsize_t(idx); + else + obj = SubString_new_object(&name); + if (obj == NULL) + goto done; + + /* return a tuple of values */ + result = PyTuple_Pack(2, is_attr_obj, obj); + + done: + Py_XDECREF(is_attr_obj); + Py_XDECREF(obj); + return result; + } +} + +static PyMethodDef fieldnameiter_methods[] = { + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject PyFieldNameIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "fieldnameiterator", /* tp_name */ + sizeof(fieldnameiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)fieldnameiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)fieldnameiter_next, /* tp_iternext */ + fieldnameiter_methods, /* tp_methods */ + 0}; + +/* unicode_formatter_field_name_split is used to implement + string.Formatter.vformat. it takes an PEP 3101 "field name", and + returns a tuple of (first, rest): "first", the part before the + first '.' or '['; and "rest", an iterator for the rest of the field + name. it's a wrapper around stringlib/string_format.h's + field_name_split. The iterator it returns is a + FieldNameIterator */ +static PyObject * +formatter_field_name_split(PyObject *ignored, PyUnicodeObject *self) +{ + SubString first; + Py_ssize_t first_idx; + fieldnameiterobject *it; + + PyObject *first_obj = NULL; + PyObject *result = NULL; + + if (!PyUnicode_Check(self)) { + PyErr_Format(PyExc_TypeError, "expected str, got %s", Py_TYPE(self)->tp_name); + return NULL; + } + + if (PyUnicode_READY(self) == -1) + return NULL; + + it = PyObject_New(fieldnameiterobject, &PyFieldNameIter_Type); + if (it == NULL) + return NULL; + + /* take ownership, give the object to the iterator. this is + just to keep the field_name alive */ + Py_INCREF(self); + it->str = self; + + /* Pass in auto_number = NULL. We'll return an empty string for + first_obj in that case. */ + if (!field_name_split((PyObject*)self, 0, PyUnicode_GET_LENGTH(self), + &first, &first_idx, &it->it_field, NULL)) + goto done; + + /* first becomes an integer, if possible; else a string */ + if (first_idx != -1) + first_obj = PyLong_FromSsize_t(first_idx); + else + /* convert "first" into a string object */ + first_obj = SubString_new_object(&first); + if (first_obj == NULL) + goto done; + + /* return a tuple of values */ + result = PyTuple_Pack(2, first_obj, it); + +done: + Py_XDECREF(it); + Py_XDECREF(first_obj); + return result; +} diff --git a/Objects/stringlib/unicodedefs.h b/Objects/stringlib/unicodedefs.h --- a/Objects/stringlib/unicodedefs.h +++ b/Objects/stringlib/unicodedefs.h @@ -6,6 +6,8 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 1 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyUnicodeObject #define STRINGLIB_CHAR Py_UNICODE #define STRINGLIB_TYPE_NAME "unicode" diff --git a/Objects/typeobject.c b/Objects/typeobject.c --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -20,10 +20,11 @@ >> (8*sizeof(unsigned int) - MCACHE_SIZE_EXP)) #define MCACHE_HASH_METHOD(type, name) \ MCACHE_HASH((type)->tp_version_tag, \ - ((PyUnicodeObject *)(name))->hash) + ((PyASCIIObject *)(name))->hash) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ - PyUnicode_GET_SIZE(name) <= MCACHE_MAX_ATTR_SIZE + PyUnicode_READY(name) != -1 && \ + PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE struct method_cache_entry { unsigned int version; @@ -3489,7 +3490,7 @@ if (self_as_str != NULL) { /* Issue 7994: If we're converting to a string, we should reject format specifications */ - if (PyUnicode_GET_SIZE(format_spec) > 0) { + if (PyUnicode_GET_LENGTH(format_spec) > 0) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "object.__format__ with a non-empty format " "string is deprecated", 1) < 0) { @@ -5122,14 +5123,21 @@ return res; } else { - PyObject *ress; + /* PyObject *ress; */ PyErr_Clear(); res = slot_tp_repr(self); if (!res) return NULL; + /* XXX this is non-sensical. Why should we return + a bytes object from __str__. Is this code even + used? - mvl */ + assert(0); + return res; + /* ress = _PyUnicode_AsDefaultEncodedString(res); Py_DECREF(res); return ress; + */ } } @@ -6206,7 +6214,7 @@ /* We want __class__ to return the class of the super object (i.e. super, or a subclass), not the class of su->obj. */ skip = (PyUnicode_Check(name) && - PyUnicode_GET_SIZE(name) == 9 && + PyUnicode_GET_LENGTH(name) == 9 && PyUnicode_CompareWithASCIIString(name, "__class__") == 0); } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -90,6 +90,19 @@ extern "C" { #endif +#define _PyUnicode_WSTR(op) (((PyASCIIObject*)(op))->wstr) +#define _PyUnicode_WSTR_LENGTH(op) (((PyCompactUnicodeObject*)(op))->wstr_length) +#define _PyUnicode_LENGTH(op) (((PyASCIIObject *)(op))->length) +#define _PyUnicode_STATE(op) (((PyASCIIObject *)(op))->state) +#define _PyUnicode_HASH(op) (((PyASCIIObject *)(op))->hash) +#define _PyUnicode_KIND(op) \ + (assert(PyUnicode_Check(op)), \ + ((PyASCIIObject *)(op))->state.kind) +#define _PyUnicode_GET_LENGTH(op) \ + (assert(PyUnicode_Check(op)), \ + ((PyASCIIObject *)(op))->length) + + /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -100,10 +113,6 @@ */ static PyObject *interned; -/* Free list for Unicode objects */ -static PyUnicodeObject *free_list; -static int numfree; - /* The empty Unicode object is shared to improve performance. */ static PyUnicodeObject *unicode_empty; @@ -226,7 +235,7 @@ (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch))) Py_LOCAL_INLINE(BLOOM_MASK) -make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len) +make_bloom_mask(int kind, void* ptr, Py_ssize_t len) { /* calculate simple bloom-style bitmask for a given unicode string */ @@ -235,36 +244,58 @@ mask = 0; for (i = 0; i < len; i++) - BLOOM_ADD(mask, ptr[i]); + BLOOM_ADD(mask, PyUnicode_READ(kind, ptr, i)); return mask; } -Py_LOCAL_INLINE(int) -unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen) -{ +#define BLOOM_MEMBER(mask, chr, str) \ + (BLOOM(mask, chr) \ + && (PyUnicode_FindChar(str, chr, 0, PyUnicode_GET_LENGTH(str), 1) >= 0)) + +/* --- Unicode Object ----------------------------------------------------- */ + +static PyObject * +substring(PyUnicodeObject *self, Py_ssize_t start, Py_ssize_t len); + +static PyObject * +fixup(PyUnicodeObject *self, Py_UCS4 (*fixfct)(PyUnicodeObject *s)); + +Py_LOCAL_INLINE(char *) findchar(void *s, int kind, + Py_ssize_t size, Py_UCS4 ch, + int direction) +{ + /* like wcschr, but doesn't stop at NULL characters */ Py_ssize_t i; - - for (i = 0; i < setlen; i++) - if (set[i] == chr) - return 1; - - return 0; -} - -#define BLOOM_MEMBER(mask, chr, set, setlen) \ - BLOOM(mask, chr) && unicode_member(chr, set, setlen) - -/* --- Unicode Object ----------------------------------------------------- */ + if (direction == 1) { + for(i = 0; i < size; i++) + if (PyUnicode_READ(kind, s, i) == ch) + return (char*)s + PyUnicode_KIND_SIZE(kind, i); + } + else { + for(i = size-1; i >= 0; i--) + if (PyUnicode_READ(kind, s, i) == ch) + return (char*)s + PyUnicode_KIND_SIZE(kind, i); + } + return NULL; +} static int unicode_resize(register PyUnicodeObject *unicode, - Py_ssize_t length) + Py_ssize_t length) { void *oldstr; + /* Resizing is only supported for old unicode objects. */ + assert(!PyUnicode_IS_COMPACT(unicode)); + assert(_PyUnicode_WSTR(unicode) != NULL); + + /* ... and only if they have not been readied yet, because + callees usually rely on the wstr representation when resizing. */ + assert(unicode->data.any == NULL); + /* Shortcut if there's nothing much to do. */ - if (unicode->length == length) + if (_PyUnicode_WSTR_LENGTH(unicode) == length) goto reset; /* Resizing shared object (unicode_empty or single character @@ -272,9 +303,9 @@ instead ! */ if (unicode == unicode_empty || - (unicode->length == 1 && - unicode->str[0] < 256U && - unicode_latin1[unicode->str[0]] == unicode)) { + (_PyUnicode_WSTR_LENGTH(unicode) == 1 && + _PyUnicode_WSTR(unicode)[0] < 256U && + unicode_latin1[_PyUnicode_WSTR(unicode)[0]] == unicode)) { PyErr_SetString(PyExc_SystemError, "can't resize shared str objects"); return -1; @@ -285,23 +316,31 @@ safe to look at str[length] (without making any assumptions about what it contains). */ - oldstr = unicode->str; - unicode->str = PyObject_REALLOC(unicode->str, - sizeof(Py_UNICODE) * (length + 1)); - if (!unicode->str) { - unicode->str = (Py_UNICODE *)oldstr; + oldstr = _PyUnicode_WSTR(unicode); + _PyUnicode_WSTR(unicode) = PyObject_REALLOC(_PyUnicode_WSTR(unicode), + sizeof(Py_UNICODE) * (length + 1)); + if (!_PyUnicode_WSTR(unicode)) { + _PyUnicode_WSTR(unicode) = (Py_UNICODE *)oldstr; PyErr_NoMemory(); return -1; } - unicode->str[length] = 0; - unicode->length = length; + _PyUnicode_WSTR(unicode)[length] = 0; + _PyUnicode_WSTR_LENGTH(unicode) = length; reset: - /* Reset the object caches */ - if (unicode->defenc) { - Py_CLEAR(unicode->defenc); - } - unicode->hash = -1; + if (unicode->data.any != NULL) { + PyObject_FREE(unicode->data.any); + if (unicode->_base.utf8 && unicode->_base.utf8 != unicode->data.any) { + PyObject_FREE(unicode->_base.utf8); + } + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + unicode->data.any = NULL; + _PyUnicode_LENGTH(unicode) = 0; + _PyUnicode_STATE(unicode).interned = _PyUnicode_STATE(unicode).interned; + _PyUnicode_STATE(unicode).kind = PyUnicode_WCHAR_KIND; + } + _PyUnicode_HASH(unicode) = -1; return 0; } @@ -315,10 +354,15 @@ */ +#ifdef Py_DEBUG +int unicode_old_new_calls = 0; +#endif + static PyUnicodeObject * _PyUnicode_New(Py_ssize_t length) { register PyUnicodeObject *unicode; + size_t new_size; /* Optimization for empty strings */ if (length == 0 && unicode_empty != NULL) { @@ -330,40 +374,26 @@ if (length > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { return (PyUnicodeObject *)PyErr_NoMemory(); } - - /* Unicode freelist & memory allocation */ - if (free_list) { - unicode = free_list; - free_list = *(PyUnicodeObject **)unicode; - numfree--; - if (unicode->str) { - /* Keep-Alive optimization: we only upsize the buffer, - never downsize it. */ - if ((unicode->length < length) && - unicode_resize(unicode, length) < 0) { - PyObject_DEL(unicode->str); - unicode->str = NULL; - } - } - else { - size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); - unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); - } - PyObject_INIT(unicode, &PyUnicode_Type); - } - else { - size_t new_size; - unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); - if (unicode == NULL) - return NULL; - new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); - unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); - } - - if (!unicode->str) { + if (length < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to _PyUnicode_New"); + return NULL; + } + +#ifdef Py_DEBUG + ++unicode_old_new_calls; +#endif + + unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); + if (unicode == NULL) + return NULL; + new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); + _PyUnicode_WSTR(unicode) = (Py_UNICODE*) PyObject_MALLOC(new_size); + if (!_PyUnicode_WSTR(unicode)) { PyErr_NoMemory(); goto onError; } + /* Initialize the first element to guard against cases where * the caller fails before initializing str -- unicode_resize() * reads str[0], and the Keep-Alive optimization can keep memory @@ -371,12 +401,19 @@ * We don't want unicode_resize to read uninitialized memory in * that case. */ - unicode->str[0] = 0; - unicode->str[length] = 0; - unicode->length = length; - unicode->hash = -1; - unicode->state = 0; - unicode->defenc = NULL; + _PyUnicode_WSTR(unicode)[0] = 0; + _PyUnicode_WSTR(unicode)[length] = 0; + _PyUnicode_WSTR_LENGTH(unicode) = length; + _PyUnicode_HASH(unicode) = -1; + _PyUnicode_STATE(unicode).interned = 0; + _PyUnicode_STATE(unicode).kind = 0; + _PyUnicode_STATE(unicode).compact = 0; + _PyUnicode_STATE(unicode).ready = 0; + _PyUnicode_STATE(unicode).ascii = 0; + unicode->data.any = NULL; + _PyUnicode_LENGTH(unicode) = 0; + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; return unicode; onError: @@ -387,6 +424,467 @@ return NULL; } +#ifdef Py_DEBUG +int unicode_new_new_calls = 0; + +/* Functions wrapping macros for use in debugger */ +char *_PyUnicode_utf8(void *unicode){ + return _PyUnicode_UTF8(unicode); +} + +void *_PyUnicode_compact_data(void *unicode) { + return _PyUnicode_COMPACT_DATA(unicode); +} +void *_PyUnicode_data(void *unicode){ + printf("obj %p\n", unicode); + printf("compact %d\n", PyUnicode_IS_COMPACT(unicode)); + printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode)); + printf("ascii op %p\n", ((void*)((PyASCIIObject*)(unicode) + 1))); + printf("compact op %p\n", ((void*)((PyCompactUnicodeObject*)(unicode) + 1))); + printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode)); + return PyUnicode_DATA(unicode); +} +#endif + +PyObject * +PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) +{ + PyObject *obj; + PyCompactUnicodeObject *unicode; + void *data; + int kind_state; + int is_sharing = 0, is_ascii = 0; + Py_ssize_t char_size; + Py_ssize_t struct_size; + + /* Optimization for empty strings */ + if (size == 0 && unicode_empty != NULL) { + Py_INCREF(unicode_empty); + return (PyObject *)unicode_empty; + } + +#ifdef Py_DEBUG + ++unicode_new_new_calls; +#endif + + struct_size = sizeof(PyCompactUnicodeObject); + if (maxchar < 128) { + kind_state = PyUnicode_1BYTE_KIND; + char_size = 1; + is_ascii = 1; + struct_size = sizeof(PyASCIIObject); + } + else if (maxchar < 256) { + kind_state = PyUnicode_1BYTE_KIND; + char_size = 1; + } + else if (maxchar < 65536) { + kind_state = PyUnicode_2BYTE_KIND; + char_size = 2; + if (sizeof(wchar_t) == 2) + is_sharing = 1; + } + else { + kind_state = PyUnicode_4BYTE_KIND; + char_size = 4; + if (sizeof(wchar_t) == 4) + is_sharing = 1; + } + + /* Ensure we won't overflow the size. */ + if (size < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to PyUnicode_New"); + return NULL; + } + if (size > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) + return PyErr_NoMemory(); + + /* Duplicated allocation code from _PyObject_New() instead of a call to + * PyObject_New() so we are able to allocate space for the object and + * it's data buffer. + */ + obj = (PyObject *) PyObject_MALLOC(struct_size + (size + 1) * char_size); + if (obj == NULL) + return PyErr_NoMemory(); + obj = PyObject_INIT(obj, &PyUnicode_Type); + if (obj == NULL) + return NULL; + + unicode = (PyCompactUnicodeObject *)obj; + if (is_ascii) + data = ((PyASCIIObject*)obj) + 1; + else + data = unicode + 1; + _PyUnicode_LENGTH(unicode) = size; + _PyUnicode_HASH(unicode) = -1; + _PyUnicode_STATE(unicode).interned = 0; + _PyUnicode_STATE(unicode).kind = kind_state; + _PyUnicode_STATE(unicode).compact = 1; + _PyUnicode_STATE(unicode).ready = 1; + _PyUnicode_STATE(unicode).ascii = is_ascii; + if (is_ascii) { + ((char*)data)[size] = 0; + _PyUnicode_WSTR(unicode) = NULL; + } + else if (kind_state == PyUnicode_1BYTE_KIND) { + ((char*)data)[size] = 0; + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; + unicode->utf8_length = 0; + unicode->utf8 = NULL; + } + else { + unicode->utf8 = NULL; + if (kind_state == PyUnicode_2BYTE_KIND) + ((Py_UCS2*)data)[size] = 0; + else /* kind_state == PyUnicode_4BYTE_KIND */ + ((Py_UCS4*)data)[size] = 0; + if (is_sharing) { + _PyUnicode_WSTR_LENGTH(unicode) = size; + _PyUnicode_WSTR(unicode) = (wchar_t *)data; + } + else { + _PyUnicode_WSTR_LENGTH(unicode) = 0; + _PyUnicode_WSTR(unicode) = NULL; + } + } + return obj; +} + +#if SIZEOF_WCHAR_T == 2 +/* Helper function to convert a 16-bits wchar_t representation to UCS4, this + will decode surrogate pairs, the other conversions are implemented as macros + for efficency. + + This function assumes that unicode can hold one more code point than wstr + characters for a terminating null character. */ +static int +unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, + PyUnicodeObject *unicode) +{ + const wchar_t *iter; + Py_UCS4 *ucs4_out; + + assert(unicode && PyUnicode_Check(unicode)); + assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); + ucs4_out = PyUnicode_4BYTE_DATA(unicode); + + for (iter = begin; iter < end; ) { + assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + + _PyUnicode_GET_LENGTH(unicode))); + if (*iter >= 0xD800 && *iter <= 0xDBFF + && (iter+1) < end && iter[1] >= 0xDC00 && iter[1] <= 0xDFFF) + { + *ucs4_out++ = (((iter[0] & 0x3FF)<<10) | (iter[1] & 0x3FF)) + 0x10000; + iter += 2; + } + else { + *ucs4_out++ = *iter; + iter++; + } + } + assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + + _PyUnicode_GET_LENGTH(unicode))); + + return 0; +} +#endif + +int +PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, + Py_ssize_t how_many) +{ + int from_kind; + int to_kind; + + assert(PyUnicode_Check(from)); + assert(PyUnicode_Check(to)); + + if (PyUnicode_READY(from)) + return -1; + if (PyUnicode_READY(to)) + return -1; + + from_kind = PyUnicode_KIND(from); + to_kind = PyUnicode_KIND(to); + + if (from_kind == to_kind) { + const Py_ssize_t char_size = PyUnicode_CHARACTER_SIZE(to); + Py_MEMCPY(PyUnicode_1BYTE_DATA(to) + (to_start * char_size), + PyUnicode_1BYTE_DATA(from) + (from_start * char_size), + how_many * char_size); + return 0; + } + + switch (from_kind) { + case PyUnicode_1BYTE_KIND: + switch (to_kind) { + case PyUnicode_2BYTE_KIND: + PyUnicode_CONVERT_BYTES( + unsigned char, Py_UCS2, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_2BYTE_DATA(to) + to_start + ); + break; + case PyUnicode_4BYTE_KIND: + PyUnicode_CONVERT_BYTES( + unsigned char, Py_UCS4, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + break; + default: + goto invalid_state; + } + break; + case PyUnicode_2BYTE_KIND: + switch (to_kind) { + case PyUnicode_1BYTE_KIND: + PyUnicode_CONVERT_BYTES( + Py_UCS2, unsigned char, + PyUnicode_2BYTE_DATA(from) + from_start, + PyUnicode_2BYTE_DATA(from) + from_start + how_many, + PyUnicode_1BYTE_DATA(to) + to_start + ); + break; + case PyUnicode_4BYTE_KIND: + PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS4, + PyUnicode_2BYTE_DATA(from) + from_start, + PyUnicode_2BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + break; + default: + goto invalid_state; + } + break; + case PyUnicode_4BYTE_KIND: + switch (to_kind) { + case PyUnicode_1BYTE_KIND: + PyUnicode_CONVERT_BYTES( + Py_UCS4, unsigned char, + PyUnicode_4BYTE_DATA(from) + from_start, + PyUnicode_4BYTE_DATA(from) + from_start + how_many, + PyUnicode_1BYTE_DATA(to) + to_start + ); + break; + case PyUnicode_2BYTE_KIND: + PyUnicode_CONVERT_BYTES( + Py_UCS4, Py_UCS2, + PyUnicode_4BYTE_DATA(from) + from_start, + PyUnicode_4BYTE_DATA(from) + from_start + how_many, + PyUnicode_2BYTE_DATA(to) + to_start + ); + break; + default: + goto invalid_state; + } + break; + default: + goto invalid_state; + } + return 0; + +invalid_state: + PyErr_Format(PyExc_ValueError, + "Impossible kind state (from=%i, to=%i) " + "in PyUnicode_CopyCharacters", + from_kind, to_kind); + return -1; +} + +int +_PyUnicode_FindMaxCharAndNumSurrogatePairs(const wchar_t *begin, + const wchar_t *end, + Py_UCS4 *maxchar, + Py_ssize_t *num_surrogates) +{ + const wchar_t *iter; + + if (num_surrogates == NULL || maxchar == NULL) { + PyErr_SetString(PyExc_SystemError, + "unexpected NULL arguments to " + "PyUnicode_FindMaxCharAndNumSurrogatePairs"); + return -1; + } + + *num_surrogates = 0; + *maxchar = 0; + + for (iter = begin; iter < end; ) { + if (*iter > *maxchar) + *maxchar = *iter; +#if SIZEOF_WCHAR_T == 2 + if (*iter >= 0xD800 && *iter <= 0xDBFF + && (iter+1) < end && iter[1] >= 0xDC00 && iter[1] <= 0xDFFF) + { + Py_UCS4 surrogate_val; + surrogate_val = (((iter[0] & 0x3FF)<<10) + | (iter[1] & 0x3FF)) + 0x10000; + ++(*num_surrogates); + if (surrogate_val > *maxchar) + *maxchar = surrogate_val; + iter += 2; + } + else + iter++; +#else + iter++; +#endif + } + return 0; +} + +#ifdef Py_DEBUG +int unicode_ready_calls = 0; +#endif + +int +_PyUnicode_Ready(PyUnicodeObject *unicode) +{ + wchar_t *end; + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; +#if SIZEOF_WCHAR_T == 2 + Py_ssize_t length_wo_surrogates; +#endif + + assert(PyUnicode_Check(unicode)); + + if (unicode->data.any != NULL) { + assert(PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND); + return 0; + } + + /* _PyUnicode_Ready() is only intented for old-style API usage where + * strings were created using _PyObject_New() and where no canonical + * representation (the str field) has been set yet aka strings + * which are not yet ready. + */ + assert(_PyUnicode_WSTR(unicode) != NULL); + assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND); + assert(!PyUnicode_IS_COMPACT(unicode)); + assert(!PyUnicode_IS_READY(unicode)); + /* Actually, it should neither be interned nor be anything else: */ + assert(_PyUnicode_STATE(unicode).interned == 0); + assert(unicode->_base.utf8 == NULL); + +#ifdef Py_DEBUG + ++unicode_ready_calls; +#endif + + end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode); + if (_PyUnicode_FindMaxCharAndNumSurrogatePairs(_PyUnicode_WSTR(unicode), end, + &maxchar, + &num_surrogates) == -1) { + assert(0 && "PyUnicode_FindMaxCharAndNumSurrogatePairs failed"); + return -1; + } + + if (maxchar < 256) { + unicode->data.any = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1); + if (!unicode->data.any) { + PyErr_NoMemory(); + return -1; + } + PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, + _PyUnicode_WSTR(unicode), end, + PyUnicode_1BYTE_DATA(unicode)); + PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_1BYTE_KIND; + if (maxchar < 128) { + unicode->_base.utf8 = unicode->data.any; + unicode->_base.utf8_length = _PyUnicode_WSTR_LENGTH(unicode); + } + else { + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + } + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; + } + /* In this case we might have to convert down from 4-byte native + wchar_t to 2-byte unicode. */ + else if (maxchar < 65536) { + assert(num_surrogates == 0 && + "FindMaxCharAndNumSurrogatePairs() messed up"); + + if (sizeof(wchar_t) == 2) { + /* We can share representations and are done. */ + unicode->data.any = _PyUnicode_WSTR(unicode); + PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + } + else { + assert(sizeof(wchar_t) == 4); + + unicode->data.any = PyObject_MALLOC( + 2 * (_PyUnicode_WSTR_LENGTH(unicode) + 1)); + if (!unicode->data.any) { + PyErr_NoMemory(); + return -1; + } + PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, + _PyUnicode_WSTR(unicode), end, + PyUnicode_2BYTE_DATA(unicode)); + PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; + } + } + /* maxchar exeeds 16 bit, wee need 4 bytes for unicode characters */ + else { +#if SIZEOF_WCHAR_T == 2 + /* in case the native representation is 2-bytes, we need to allocate a + new normalized 4-byte version. */ + length_wo_surrogates = _PyUnicode_WSTR_LENGTH(unicode) - num_surrogates; + unicode->data.any = PyObject_MALLOC(4 * (length_wo_surrogates + 1)); + if (!unicode->data.any) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_LENGTH(unicode) = length_wo_surrogates; + _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + if (unicode_convert_wchar_to_ucs4(_PyUnicode_WSTR(unicode), end, + unicode) < 0) { + assert(0 && "ConvertWideCharToUCS4 failed"); + return -1; + } + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; +#else + assert(num_surrogates == 0); + + unicode->data.any = _PyUnicode_WSTR(unicode); + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + unicode->_base.utf8 = NULL; + unicode->_base.utf8_length = 0; + _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; +#endif + PyUnicode_4BYTE_DATA(unicode)[_PyUnicode_LENGTH(unicode)] = '\0'; + } + _PyUnicode_STATE(unicode).ready = 1; + return 0; +} + static void unicode_dealloc(register PyUnicodeObject *unicode) { @@ -409,25 +907,19 @@ Py_FatalError("Inconsistent interned string state."); } - if (PyUnicode_CheckExact(unicode) && - numfree < PyUnicode_MAXFREELIST) { - /* Keep-Alive optimization */ - if (unicode->length >= KEEPALIVE_SIZE_LIMIT) { - PyObject_DEL(unicode->str); - unicode->str = NULL; - unicode->length = 0; - } - if (unicode->defenc) { - Py_CLEAR(unicode->defenc); - } - /* Add to free list */ - *(PyUnicodeObject **)unicode = free_list; - free_list = unicode; - numfree++; + if (_PyUnicode_WSTR(unicode) && + (!PyUnicode_IS_READY(unicode) || + _PyUnicode_WSTR(unicode) != PyUnicode_DATA(unicode))) + PyObject_DEL(_PyUnicode_WSTR(unicode)); + if (_PyUnicode_UTF8(unicode) && _PyUnicode_UTF8(unicode) != PyUnicode_DATA(unicode)) + PyObject_DEL(unicode->_base.utf8); + + if (PyUnicode_IS_COMPACT(unicode)) { + Py_TYPE(unicode)->tp_free((PyObject *)unicode); } else { - PyObject_DEL(unicode->str); - Py_XDECREF(unicode->defenc); + if (unicode->data.any) + PyObject_DEL(unicode->data.any); Py_TYPE(unicode)->tp_free((PyObject *)unicode); } } @@ -443,21 +935,26 @@ return -1; } v = *unicode; - if (v == NULL || !PyUnicode_Check(v) || Py_REFCNT(v) != 1 || length < 0) { + if (v == NULL || !PyUnicode_Check(v) || Py_REFCNT(v) != 1 || length < 0 || + PyUnicode_IS_COMPACT(v) || _PyUnicode_WSTR(v) == NULL) { PyErr_BadInternalCall(); return -1; } /* Resizing unicode_empty and single character objects is not - possible since these are being shared. We simply return a fresh - copy with the same Unicode content. */ - if (v->length != length && - (v == unicode_empty || v->length == 1)) { + possible since these are being shared. + The same goes for new-representation unicode objects or objects which + have already been readied. + For these, we simply return a fresh copy with the same Unicode content. + */ + if ((_PyUnicode_WSTR_LENGTH(v) != length && + (v == unicode_empty || _PyUnicode_WSTR_LENGTH(v) == 1)) || + PyUnicode_IS_COMPACT(v) || v->data.any) { PyUnicodeObject *w = _PyUnicode_New(length); if (w == NULL) return -1; - Py_UNICODE_COPY(w->str, v->str, - length < v->length ? length : v->length); + Py_UNICODE_COPY(_PyUnicode_WSTR(w), _PyUnicode_WSTR(v), + length < _PyUnicode_WSTR_LENGTH(v) ? length : _PyUnicode_WSTR_LENGTH(v)); Py_DECREF(*unicode); *unicode = w; return 0; @@ -474,44 +971,85 @@ return _PyUnicode_Resize((PyUnicodeObject **)unicode, length); } +static PyObject* +get_latin1_char(unsigned char ch) +{ + PyUnicodeObject *unicode = unicode_latin1[ch]; + if (!unicode) { + unicode = (PyUnicodeObject *)PyUnicode_New(1, ch); + if (!unicode) + return NULL; + PyUnicode_1BYTE_DATA(unicode)[0] = ch; + unicode_latin1[ch] = unicode; + } + Py_INCREF(unicode); + return (PyObject *)unicode; +} + PyObject * PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) { PyUnicodeObject *unicode; + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; + + if (u == NULL) + return (PyObject*)_PyUnicode_New(size); /* If the Unicode data is known at construction time, we can apply some optimizations which share commonly used objects. */ - if (u != NULL) { - - /* Optimization for empty strings */ - if (size == 0 && unicode_empty != NULL) { - Py_INCREF(unicode_empty); - return (PyObject *)unicode_empty; - } - - /* Single character Unicode objects in the Latin-1 range are - shared when using this constructor */ - if (size == 1 && *u < 256) { - unicode = unicode_latin1[*u]; - if (!unicode) { - unicode = _PyUnicode_New(1); - if (!unicode) - return NULL; - unicode->str[0] = *u; - unicode_latin1[*u] = unicode; - } - Py_INCREF(unicode); - return (PyObject *)unicode; - } - } - - unicode = _PyUnicode_New(size); + + /* Optimization for empty strings */ + if (size == 0 && unicode_empty != NULL) { + Py_INCREF(unicode_empty); + return (PyObject *)unicode_empty; + } + + /* Single character Unicode objects in the Latin-1 range are + shared when using this constructor */ + if (size == 1 && *u < 256) + return get_latin1_char((unsigned char)*u); + + /* If not empty and not single character, copy the Unicode data + into the new object */ + if (_PyUnicode_FindMaxCharAndNumSurrogatePairs(u, u + size, &maxchar, + &num_surrogates) == -1) + return NULL; + + unicode = (PyUnicodeObject *) PyUnicode_New(size - num_surrogates, + maxchar); if (!unicode) return NULL; - /* Copy the Unicode data into the new object */ - if (u != NULL) - Py_UNICODE_COPY(unicode->str, u, size); + switch (PyUnicode_KIND(unicode)) { + case PyUnicode_1BYTE_KIND: + PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char, + u, u + size, PyUnicode_1BYTE_DATA(unicode)); + break; + case PyUnicode_2BYTE_KIND: +#if Py_UNICODE_SIZE == 2 + Py_MEMCPY(PyUnicode_2BYTE_DATA(unicode), u, size * 2); +#else + PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, + u, u + size, PyUnicode_2BYTE_DATA(unicode)); +#endif + break; + case PyUnicode_4BYTE_KIND: +#if SIZEOF_WCHAR_T == 2 + /* This is the only case which has to process surrogates, thus + a simple copy loop is not enough and we need a function. */ + if (unicode_convert_wchar_to_ucs4(u, u + size, unicode) < 0) { + Py_DECREF(unicode); + return NULL; + } +#else + assert(num_surrogates == 0); + Py_MEMCPY(PyUnicode_4BYTE_DATA(unicode), u, size * 4); +#endif + break; + default: + assert(0 && "Impossible state"); + } return (PyObject *)unicode; } @@ -541,18 +1079,8 @@ /* Single characters are shared when using this constructor. Restrict to ASCII, since the input must be UTF-8. */ - if (size == 1 && Py_CHARMASK(*u) < 128) { - unicode = unicode_latin1[Py_CHARMASK(*u)]; - if (!unicode) { - unicode = _PyUnicode_New(1); - if (!unicode) - return NULL; - unicode->str[0] = Py_CHARMASK(*u); - unicode_latin1[Py_CHARMASK(*u)] = unicode; - } - Py_INCREF(unicode); - return (PyObject *)unicode; - } + if (size == 1 && Py_CHARMASK(*u) < 128) + return get_latin1_char(Py_CHARMASK(*u)); return PyUnicode_DecodeUTF8(u, size, NULL); } @@ -576,28 +1104,196 @@ return PyUnicode_FromStringAndSize(u, size); } +PyObject* +PyUnicode_FromUCS1(const unsigned char* u, Py_ssize_t size) +{ + PyObject *res; + unsigned char max = 127; + Py_ssize_t i; + for (i = 0; i < size; i++) { + if (u[i] & 0x80) { + max = 255; + break; + } + } + res = PyUnicode_New(size, max); + if (!res) + return NULL; + memcpy(PyUnicode_1BYTE_DATA(res), u, size); + return res; +} + +PyObject* +PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size) +{ + PyObject *res; + Py_UCS2 max = 0; + Py_ssize_t i; + for (i = 0; i < size; i++) + if (u[i] > max) + max = u[i]; + res = PyUnicode_New(size, max); + if (!res) + return NULL; + if (max >= 256) + memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size); + else + for (i = 0; i < size; i++) + PyUnicode_1BYTE_DATA(res)[i] = (Py_UCS1)u[i]; + return res; +} + +PyObject* +PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) +{ + PyObject *res; + Py_UCS4 max = 0; + Py_ssize_t i; + for (i = 0; i < size; i++) + if (u[i] > max) + max = u[i]; + res = PyUnicode_New(size, max); + if (!res) + return NULL; + if (max >= 0x10000) + memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size); + else { + int kind = PyUnicode_KIND(res); + void *data = PyUnicode_DATA(res); + for (i = 0; i < size; i++) + PyUnicode_WRITE(kind, data, i, u[i]); + } + return res; +} + +PyObject* +PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) +{ + switch(kind) { + case PyUnicode_1BYTE_KIND: + return PyUnicode_FromUCS1(buffer, size); + case PyUnicode_2BYTE_KIND: + return PyUnicode_FromUCS2(buffer, size); + case PyUnicode_4BYTE_KIND: + return PyUnicode_FromUCS4(buffer, size); + } + assert(0); + return NULL; +} + + +/* Widen Unicode objects to larger buffers. + Return NULL if the string is too wide already. */ + +void* +_PyUnicode_AsKind(PyObject *s, unsigned int kind) +{ + Py_ssize_t i; + Py_ssize_t len = PyUnicode_GET_LENGTH(s); + void *d = PyUnicode_DATA(s); + unsigned int skind = PyUnicode_KIND(s); + if (PyUnicode_KIND(s) >= kind) { + PyErr_SetString(PyExc_RuntimeError, "invalid widening attempt"); + return NULL; + } + switch(kind) { + case PyUnicode_2BYTE_KIND: { + Py_UCS2 *result = PyMem_Malloc(PyUnicode_GET_LENGTH(s) * sizeof(Py_UCS2)); + if (!result) { + PyErr_NoMemory(); + return 0; + } + for (i = 0; i < len; i++) + result[i] = ((Py_UCS1*)d)[i]; + return result; + } + case PyUnicode_4BYTE_KIND: { + Py_UCS4 *result = PyMem_Malloc(PyUnicode_GET_LENGTH(s) * sizeof(Py_UCS4)); + if (!result) { + PyErr_NoMemory(); + return 0; + } + for (i = 0; i < len; i++) + result[i] = PyUnicode_READ(skind, d, i); + return result; + } + } + Py_FatalError("invalid kind"); + return NULL; +} + +static Py_UCS4* +as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, + int copy_null) +{ + int kind; + void *data; + Py_ssize_t len, targetlen; + if (PyUnicode_READY(string) == -1) + return NULL; + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); + len = PyUnicode_GET_LENGTH(string); + targetlen = len; + if (copy_null) + targetlen++; + if (!target) { + if (PY_SSIZE_T_MAX / sizeof(Py_UCS4) < targetlen) { + PyErr_NoMemory(); + return NULL; + } + target = PyMem_Malloc(targetlen * sizeof(Py_UCS4)); + if (!target) { + PyErr_NoMemory(); + return NULL; + } + } + else { + if (targetsize < targetlen) { + PyErr_Format(PyExc_SystemError, + "string is longer than the buffer"); + if (copy_null && 0 < targetsize) + target[0] = 0; + return NULL; + } + } + if (kind != PyUnicode_4BYTE_KIND) { + Py_ssize_t i; + for (i = 0; i < len; i++) + target[i] = PyUnicode_READ(kind, data, i); + } + else + Py_MEMCPY(target, data, len * sizeof(Py_UCS4)); + if (copy_null) + target[len] = 0; + return target; +} + +Py_UCS4* +PyUnicode_AsUCS4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, + int copy_null) +{ + if (target == NULL || targetsize < 1) { + PyErr_BadInternalCall(); + return NULL; + } + return as_ucs4(string, target, targetsize, copy_null); +} + +Py_UCS4* +PyUnicode_AsUCS4Copy(PyObject *string) +{ + return as_ucs4(string, NULL, 0, 1); +} + #ifdef HAVE_WCHAR_H -#if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) -# define CONVERT_WCHAR_TO_SURROGATES -#endif - -#ifdef CONVERT_WCHAR_TO_SURROGATES - -/* Here sizeof(wchar_t) is 4 but Py_UNICODE_SIZE == 2, so we need - to convert from UTF32 to UTF16. */ - PyObject * PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size) { - PyUnicodeObject *unicode; - register Py_ssize_t i; - Py_ssize_t alloc; - const wchar_t *orig_w; - if (w == NULL) { if (size == 0) - return PyUnicode_FromStringAndSize(NULL, 0); + return PyUnicode_New(0, 0); PyErr_BadInternalCall(); return NULL; } @@ -606,77 +1302,10 @@ size = wcslen(w); } - alloc = size; - orig_w = w; - for (i = size; i > 0; i--) { - if (*w > 0xFFFF) - alloc++; - w++; - } - w = orig_w; - unicode = _PyUnicode_New(alloc); - if (!unicode) - return NULL; - - /* Copy the wchar_t data into the new object */ - { - register Py_UNICODE *u; - u = PyUnicode_AS_UNICODE(unicode); - for (i = size; i > 0; i--) { - if (*w > 0xFFFF) { - wchar_t ordinal = *w++; - ordinal -= 0x10000; - *u++ = 0xD800 | (ordinal >> 10); - *u++ = 0xDC00 | (ordinal & 0x3FF); - } - else - *u++ = *w++; - } - } - return (PyObject *)unicode; -} - -#else - -PyObject * -PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size) -{ - PyUnicodeObject *unicode; - - if (w == NULL) { - if (size == 0) - return PyUnicode_FromStringAndSize(NULL, 0); - PyErr_BadInternalCall(); - return NULL; - } - - if (size == -1) { - size = wcslen(w); - } - - unicode = _PyUnicode_New(size); - if (!unicode) - return NULL; - - /* Copy the wchar_t data into the new object */ -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - memcpy(unicode->str, w, size * sizeof(wchar_t)); -#else - { - register Py_UNICODE *u; - register Py_ssize_t i; - u = PyUnicode_AS_UNICODE(unicode); - for (i = size; i > 0; i--) - *u++ = *w++; - } -#endif - - return (PyObject *)unicode; -} - -#endif /* CONVERT_WCHAR_TO_SURROGATES */ - -#undef CONVERT_WCHAR_TO_SURROGATES + return PyUnicode_FromUnicode(w, size); +} + +#endif /* HAVE_WCHAR_H */ static void makefmt(char *fmt, int longflag, int longlongflag, int size_tflag, @@ -779,10 +1408,6 @@ return f; } -#define appendstring(string) {for (copy = string;*copy;) *s++ = *copy++;} - -/* size of fixed-size buffer for formatting single arguments */ -#define ITEM_BUFFER_LEN 21 /* maximum number of characters required for output of %ld. 21 characters allows for 64-bit integers (in decimal) and an optional sign. */ #define MAX_LONG_CHARS 21 @@ -803,80 +1428,35 @@ int precision = 0; int zeropad; const char* f; - Py_UNICODE *s; - PyObject *string; + PyUnicodeObject *string; /* used by sprintf */ - char buffer[ITEM_BUFFER_LEN+1]; - /* use abuffer instead of buffer, if we need more space - * (which can happen if there's a format specifier with width). */ - char *abuffer = NULL; - char *realbuffer; - Py_ssize_t abuffersize = 0; char fmt[61]; /* should be enough for %0width.precisionlld */ - const char *copy; + Py_UCS4 maxchar = 127; /* result is ASCII by default */ + Py_UCS4 argmaxchar; + Py_ssize_t numbersize = 0; + char *numberresults = NULL; + char *numberresult = NULL; + Py_ssize_t i; + int kind; + void *data; Py_VA_COPY(count, vargs); /* step 1: count the number of %S/%R/%A/%s format specifications * (we call PyObject_Str()/PyObject_Repr()/PyObject_ASCII()/ * PyUnicode_DecodeUTF8() for these objects once during step 3 and put the - * result in an array) */ - for (f = format; *f; f++) { - if (*f == '%') { - /* skip width or width.precision (eg. "1.2" of "%1.2f") */ - f = parse_format_flags(f, NULL, NULL, NULL, NULL, NULL); - if (*f == 's' || *f=='S' || *f=='R' || *f=='A' || *f=='V') - ++callcount; - } - else if (128 <= (unsigned char)*f) { - PyErr_Format(PyExc_ValueError, - "PyUnicode_FromFormatV() expects an ASCII-encoded format " - "string, got a non-ASCII byte: 0x%02x", - (unsigned char)*f); - return NULL; - } - } - /* step 2: allocate memory for the results of - * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */ - if (callcount) { - callresults = PyObject_Malloc(sizeof(PyObject *)*callcount); - if (!callresults) { - PyErr_NoMemory(); - return NULL; - } - callresult = callresults; - } - /* step 3: figure out how large a buffer we need */ + * result in an array) + * also esimate a upper bound for all the number formats in the string, + * numbers will be formated in step 3 and be keept in a '\0'-separated + * buffer before putting everything together. */ for (f = format; *f; f++) { if (*f == '%') { -#ifdef HAVE_LONG_LONG int longlongflag; -#endif - const char* p; - - p = f; - f = parse_format_flags(f, &width, NULL, - NULL, &longlongflag, NULL); - - switch (*f) { - case 'c': - { -#ifndef Py_UNICODE_WIDE - int ordinal = va_arg(count, int); - if (ordinal > 0xffff) - n += 2; - else - n++; -#else - (void)va_arg(count, int); - n++; -#endif - break; - } - case '%': - n++; - break; - case 'd': case 'u': case 'i': case 'x': - (void) va_arg(count, int); + /* skip width or width.precision (eg. "1.2" of "%1.2f") */ + f = parse_format_flags(f, &width, NULL, NULL, &longlongflag, NULL); + if (*f == 's' || *f=='S' || *f=='R' || *f=='A' || *f=='V') + ++callcount; + + else if (*f == 'd' || *f=='u' || *f=='i' || *f=='x' || *f=='p') { #ifdef HAVE_LONG_LONG if (longlongflag) { if (width < MAX_LONG_LONG_CHARS) @@ -890,10 +1470,142 @@ need more (which we allocate later). */ if (width < MAX_LONG_CHARS) width = MAX_LONG_CHARS; - n += width; - /* XXX should allow for large precision here too. */ - if (abuffersize < width) - abuffersize = width; + + /* account for the size + '\0' to separate numbers + inside of the numberresults buffer */ + numbersize += (width + 1); + } + } + else if ((unsigned char)*f > 127) { + PyErr_Format(PyExc_ValueError, + "PyUnicode_FromFormatV() expects an ASCII-encoded format " + "string, got a non-ASCII byte: 0x%02x", + (unsigned char)*f); + return NULL; + } + } + /* step 2: allocate memory for the results of + * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */ + if (callcount) { + callresults = PyObject_Malloc(sizeof(PyObject *) * callcount); + if (!callresults) { + PyErr_NoMemory(); + return NULL; + } + callresult = callresults; + } + /* step 2.5: allocate memory for the results of formating numbers */ + if (numbersize) { + numberresults = PyObject_Malloc(numbersize); + if (!numberresults) { + PyErr_NoMemory(); + goto fail; + } + numberresult = numberresults; + } + + /* step 3: format numbers and figure out how large a buffer we need */ + for (f = format; *f; f++) { + if (*f == '%') { + const char* p; + int longflag; + int longlongflag; + int size_tflag; + int numprinted; + + p = f; + zeropad = (f[1] == '0'); + f = parse_format_flags(f, &width, &precision, + &longflag, &longlongflag, &size_tflag); + switch (*f) { + case 'c': + { + Py_UCS4 ordinal = va_arg(count, int); + maxchar = PY_MAX(maxchar, ordinal); + n++; + break; + } + case '%': + n++; + break; + case 'i': + case 'd': + makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, + width, precision, *f); + if (longflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, long)); +#ifdef HAVE_LONG_LONG + else if (longlongflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, PY_LONG_LONG)); +#endif + else if (size_tflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, Py_ssize_t)); + else + numprinted = sprintf(numberresult, fmt, + va_arg(count, int)); + n += numprinted; + /* advance by +1 to skip over the '\0' */ + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'u': + makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, + width, precision, 'u'); + if (longflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned long)); +#ifdef HAVE_LONG_LONG + else if (longlongflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned PY_LONG_LONG)); +#endif + else if (size_tflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, size_t)); + else + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned int)); + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'x': + makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x'); + numprinted = sprintf(numberresult, fmt, va_arg(count, int)); + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'p': + numprinted = sprintf(numberresult, "%p", va_arg(count, void*)); + /* %p is ill-defined: ensure leading 0x. */ + if (numberresult[1] == 'X') + numberresult[1] = 'x'; + else if (numberresult[1] != 'x') { + memmove(numberresult + 2, numberresult, + strlen(numberresult) + 1); + numberresult[0] = '0'; + numberresult[1] = 'x'; + numprinted += 2; + } + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); break; case 's': { @@ -902,7 +1614,11 @@ PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace"); if (!str) goto fail; - n += PyUnicode_GET_SIZE(str); + /* since PyUnicode_DecodeUTF8 returns already flexible + unicode objects, there is no need to call ready on them */ + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; break; @@ -911,7 +1627,11 @@ { PyObject *obj = va_arg(count, PyObject *); assert(obj && PyUnicode_Check(obj)); - n += PyUnicode_GET_SIZE(obj); + if (PyUnicode_READY(obj) == -1) + goto fail; + argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(obj); break; } case 'V': @@ -922,14 +1642,20 @@ assert(obj || str); assert(!obj || PyUnicode_Check(obj)); if (obj) { - n += PyUnicode_GET_SIZE(obj); + if (PyUnicode_READY(obj) == -1) + goto fail; + argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(obj); *callresult++ = NULL; } else { str_obj = PyUnicode_DecodeUTF8(str, strlen(str), "replace"); if (!str_obj) goto fail; - n += PyUnicode_GET_SIZE(str_obj); + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str_obj); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str_obj); *callresult++ = str_obj; } break; @@ -940,9 +1666,11 @@ PyObject *str; assert(obj); str = PyObject_Str(obj); - if (!str) + if (!str || PyUnicode_READY(str) == -1) goto fail; - n += PyUnicode_GET_SIZE(str); + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; break; @@ -953,9 +1681,11 @@ PyObject *repr; assert(obj); repr = PyObject_Repr(obj); - if (!repr) + if (!repr || PyUnicode_READY(repr) == -1) goto fail; - n += PyUnicode_GET_SIZE(repr); + argmaxchar = PyUnicode_MAX_CHAR_VALUE(repr); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(repr); /* Remember the repr and switch to the next slot */ *callresult++ = repr; break; @@ -966,22 +1696,15 @@ PyObject *ascii; assert(obj); ascii = PyObject_ASCII(obj); - if (!ascii) + if (!ascii || PyUnicode_READY(ascii) == -1) goto fail; - n += PyUnicode_GET_SIZE(ascii); + argmaxchar = PyUnicode_MAX_CHAR_VALUE(ascii); + maxchar = PY_MAX(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(ascii); /* Remember the repr and switch to the next slot */ *callresult++ = ascii; break; } - case 'p': - (void) va_arg(count, int); - /* maximum 64-bit pointer representation: - * 0xffffffffffffffff - * so 19 characters is enough. - * XXX I count 18 -- what's the extra for? - */ - n += 19; - break; default: /* if we stumble upon an unknown formatting code, copy the rest of @@ -996,98 +1719,65 @@ n++; } expand: - if (abuffersize > ITEM_BUFFER_LEN) { - /* add 1 for sprintf's trailing null byte */ - abuffer = PyObject_Malloc(abuffersize + 1); - if (!abuffer) { - PyErr_NoMemory(); - goto fail; - } - realbuffer = abuffer; - } - else - realbuffer = buffer; /* step 4: fill the buffer */ - /* Since we've analyzed how much space we need for the worst case, + /* Since we've analyzed how much space we need, we don't have to resize the string. There can be no errors beyond this point. */ - string = PyUnicode_FromUnicode(NULL, n); + string = (PyUnicodeObject *)PyUnicode_New(n, maxchar); if (!string) goto fail; - - s = PyUnicode_AS_UNICODE(string); + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); callresult = callresults; - - for (f = format; *f; f++) { + numberresult = numberresults; + + for (i = 0, f = format; *f; f++) { if (*f == '%') { const char* p; - int longflag; - int longlongflag; - int size_tflag; p = f; - zeropad = (f[1] == '0'); - f = parse_format_flags(f, &width, &precision, - &longflag, &longlongflag, &size_tflag); + f = parse_format_flags(f, NULL, NULL, NULL, NULL, NULL); + /* checking for == because the last argument could be a empty + string, which causes i to point to end, the assert at the end of + the loop */ + assert(i <= PyUnicode_GET_LENGTH(string)); switch (*f) { case 'c': { - int ordinal = va_arg(vargs, int); -#ifndef Py_UNICODE_WIDE - if (ordinal > 0xffff) { - ordinal -= 0x10000; - *s++ = 0xD800 | (ordinal >> 10); - *s++ = 0xDC00 | (ordinal & 0x3FF); - } else -#endif - *s++ = ordinal; + const int ordinal = va_arg(vargs, int); + PyUnicode_WRITE(kind, data, i++, ordinal); break; } case 'i': case 'd': - makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, - width, precision, *f); - if (longflag) - sprintf(realbuffer, fmt, va_arg(vargs, long)); -#ifdef HAVE_LONG_LONG - else if (longlongflag) - sprintf(realbuffer, fmt, va_arg(vargs, PY_LONG_LONG)); -#endif - else if (size_tflag) - sprintf(realbuffer, fmt, va_arg(vargs, Py_ssize_t)); + case 'u': + case 'x': + case 'p': + /* unused, since we already have the result */ + if (*f == 'p') + (void) va_arg(vargs, void *); else - sprintf(realbuffer, fmt, va_arg(vargs, int)); - appendstring(realbuffer); - break; - case 'u': - makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, - width, precision, 'u'); - if (longflag) - sprintf(realbuffer, fmt, va_arg(vargs, unsigned long)); -#ifdef HAVE_LONG_LONG - else if (longlongflag) - sprintf(realbuffer, fmt, va_arg(vargs, - unsigned PY_LONG_LONG)); -#endif - else if (size_tflag) - sprintf(realbuffer, fmt, va_arg(vargs, size_t)); - else - sprintf(realbuffer, fmt, va_arg(vargs, unsigned int)); - appendstring(realbuffer); - break; - case 'x': - makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x'); - sprintf(realbuffer, fmt, va_arg(vargs, int)); - appendstring(realbuffer); + (void) va_arg(vargs, int); + /* extract the result from numberresults and append. */ + for (; *numberresult; ++i, ++numberresult) + PyUnicode_WRITE(kind, data, i, *numberresult); + /* skip over the separating '\0' */ + assert(*numberresult == '\0'); + numberresult++; + assert(numberresult <= numberresults + numbersize); break; case 's': { /* unused, since we already have the result */ + Py_ssize_t size; (void) va_arg(vargs, char *); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult), - PyUnicode_GET_SIZE(*callresult)); - s += PyUnicode_GET_SIZE(*callresult); + size = PyUnicode_GET_LENGTH(*callresult); + assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); + PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, 0, + size); + i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); /* switch to next unicode()/repr() result */ @@ -1097,23 +1787,35 @@ case 'U': { PyObject *obj = va_arg(vargs, PyObject *); - Py_ssize_t size = PyUnicode_GET_SIZE(obj); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); - s += size; + Py_ssize_t size; + assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); + size = PyUnicode_GET_LENGTH(obj); + PyUnicode_CopyCharacters((PyObject*)string, i, + obj, 0, + size); + i += size; break; } case 'V': { + Py_ssize_t size; PyObject *obj = va_arg(vargs, PyObject *); va_arg(vargs, const char *); if (obj) { - Py_ssize_t size = PyUnicode_GET_SIZE(obj); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); - s += size; + size = PyUnicode_GET_LENGTH(obj); + assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); + PyUnicode_CopyCharacters((PyObject*)string, i, + obj, 0, + size); + i += size; } else { - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult), - PyUnicode_GET_SIZE(*callresult)); - s += PyUnicode_GET_SIZE(*callresult); + size = PyUnicode_GET_LENGTH(*callresult); + assert(PyUnicode_KIND(*callresult) <= + PyUnicode_KIND(string)); + PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, + 0, size); + i += size; Py_DECREF(*callresult); } ++callresult; @@ -1123,52 +1825,42 @@ case 'R': case 'A': { - Py_UNICODE *ucopy; - Py_ssize_t usize; - Py_ssize_t upos; /* unused, since we already have the result */ (void) va_arg(vargs, PyObject *); - ucopy = PyUnicode_AS_UNICODE(*callresult); - usize = PyUnicode_GET_SIZE(*callresult); - for (upos = 0; upos<usize;) - *s++ = ucopy[upos++]; + assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); + PyUnicode_CopyCharacters((PyObject*)string, i, + *callresult, 0, + PyUnicode_GET_LENGTH(*callresult)); + i += PyUnicode_GET_LENGTH(*callresult); /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); /* switch to next unicode()/repr() result */ ++callresult; break; } - case 'p': - sprintf(buffer, "%p", va_arg(vargs, void*)); - /* %p is ill-defined: ensure leading 0x. */ - if (buffer[1] == 'X') - buffer[1] = 'x'; - else if (buffer[1] != 'x') { - memmove(buffer+2, buffer, strlen(buffer)+1); - buffer[0] = '0'; - buffer[1] = 'x'; - } - appendstring(buffer); - break; case '%': - *s++ = '%'; + PyUnicode_WRITE(kind, data, i++, '%'); break; default: - appendstring(p); + for (; *p; ++p, ++i) + PyUnicode_WRITE(kind, data, i, *p); + assert(i == PyUnicode_GET_LENGTH(string)); goto end; } } - else - *s++ = *f; - } + else { + assert(i < PyUnicode_GET_LENGTH(string)); + PyUnicode_WRITE(kind, data, i++, *f); + } + } + assert(i == PyUnicode_GET_LENGTH(string)); end: if (callresults) PyObject_Free(callresults); - if (abuffer) - PyObject_Free(abuffer); - PyUnicode_Resize(&string, s - PyUnicode_AS_UNICODE(string)); - return string; + if (numberresults) + PyObject_Free(numberresults); + return (PyObject *)string; fail: if (callresults) { PyObject **callresult2 = callresults; @@ -1178,13 +1870,11 @@ } PyObject_Free(callresults); } - if (abuffer) - PyObject_Free(abuffer); + if (numberresults) + PyObject_Free(numberresults); return NULL; } -#undef appendstring - PyObject * PyUnicode_FromFormat(const char *format, ...) { @@ -1201,6 +1891,8 @@ return ret; } +#ifdef HAVE_WCHAR_H + /* Helper function for PyUnicode_AsWideChar() and PyUnicode_AsWideCharString(): convert a Unicode object to a wide character string. @@ -1215,99 +1907,23 @@ wchar_t *w, Py_ssize_t size) { -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T Py_ssize_t res; + const wchar_t *wstr; + + wstr = PyUnicode_AsUnicodeAndSize((PyObject *)unicode, &res); + if (wstr == NULL) + return -1; + if (w != NULL) { - res = PyUnicode_GET_SIZE(unicode); if (size > res) size = res + 1; else res = size; - memcpy(w, unicode->str, size * sizeof(wchar_t)); + Py_MEMCPY(w, wstr, size * sizeof(wchar_t)); return res; } else - return PyUnicode_GET_SIZE(unicode) + 1; -#elif Py_UNICODE_SIZE == 2 && SIZEOF_WCHAR_T == 4 - register const Py_UNICODE *u; - const Py_UNICODE *uend; - const wchar_t *worig, *wend; - Py_ssize_t nchar; - - u = PyUnicode_AS_UNICODE(unicode); - uend = u + PyUnicode_GET_SIZE(unicode); - if (w != NULL) { - worig = w; - wend = w + size; - while (u != uend && w != wend) { - if (0xD800 <= u[0] && u[0] <= 0xDBFF - && 0xDC00 <= u[1] && u[1] <= 0xDFFF) - { - *w = (((u[0] & 0x3FF) << 10) | (u[1] & 0x3FF)) + 0x10000; - u += 2; - } - else { - *w = *u; - u++; - } - w++; - } - if (w != wend) - *w = L'\0'; - return w - worig; - } - else { - nchar = 1; /* null character at the end */ - while (u != uend) { - if (0xD800 <= u[0] && u[0] <= 0xDBFF - && 0xDC00 <= u[1] && u[1] <= 0xDFFF) - u += 2; - else - u++; - nchar++; - } - } - return nchar; -#elif Py_UNICODE_SIZE == 4 && SIZEOF_WCHAR_T == 2 - register Py_UNICODE *u, *uend, ordinal; - register Py_ssize_t i; - wchar_t *worig, *wend; - Py_ssize_t nchar; - - u = PyUnicode_AS_UNICODE(unicode); - uend = u + PyUnicode_GET_SIZE(u); - if (w != NULL) { - worig = w; - wend = w + size; - while (u != uend && w != wend) { - ordinal = *u; - if (ordinal > 0xffff) { - ordinal -= 0x10000; - *w++ = 0xD800 | (ordinal >> 10); - *w++ = 0xDC00 | (ordinal & 0x3FF); - } - else - *w++ = ordinal; - u++; - } - if (w != wend) - *w = 0; - return w - worig; - } - else { - nchar = 1; /* null character */ - while (u != uend) { - if (*u > 0xffff) - nchar += 2; - else - nchar++; - u++; - } - return nchar; - } -#else -# error "unsupported wchar_t and Py_UNICODE sizes, see issue #8670" -#endif + return res + 1; } Py_ssize_t @@ -1335,6 +1951,8 @@ } buflen = unicode_aswidechar((PyUnicodeObject *)unicode, NULL, 0); + if (buflen == -1) + return NULL; if (PY_SSIZE_T_MAX / sizeof(wchar_t) < buflen) { PyErr_NoMemory(); return NULL; @@ -1346,35 +1964,33 @@ return NULL; } buflen = unicode_aswidechar((PyUnicodeObject *)unicode, buffer, buflen); + if (buflen == -1) + return NULL; if (size != NULL) *size = buflen; return buffer; } -#endif +#endif /* HAVE_WCHAR_H */ PyObject * PyUnicode_FromOrdinal(int ordinal) { - Py_UNICODE s[2]; - + PyObject *v; if (ordinal < 0 || ordinal > 0x10ffff) { PyErr_SetString(PyExc_ValueError, "chr() arg not in range(0x110000)"); return NULL; } -#ifndef Py_UNICODE_WIDE - if (ordinal > 0xffff) { - ordinal -= 0x10000; - s[0] = 0xD800 | (ordinal >> 10); - s[1] = 0xDC00 | (ordinal & 0x3FF); - return PyUnicode_FromUnicode(s, 2); - } -#endif - - s[0] = (Py_UNICODE)ordinal; - return PyUnicode_FromUnicode(s, 1); + if (ordinal < 256) + return get_latin1_char(ordinal); + + v = PyUnicode_New(1, ordinal); + if (v == NULL) + return NULL; + PyUnicode_WRITE(PyUnicode_KIND(v), PyUnicode_DATA(v), 0, ordinal); + return v; } PyObject * @@ -1389,8 +2005,9 @@ if (PyUnicode_Check(obj)) { /* For a Unicode subtype that's not a Unicode object, return a true Unicode object with the same data. */ - return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj)); + if (PyUnicode_READY(obj) == -1) + return NULL; + return substring((PyUnicodeObject *)obj, 0, PyUnicode_GET_LENGTH(obj)); } PyErr_Format(PyExc_TypeError, "Can't convert '%.100s' object to str implicitly", @@ -1536,6 +2153,10 @@ goto onError; } Py_DECREF(buffer); + if (PyUnicode_READY(unicode)) { + Py_DECREF(unicode); + return NULL; + } return unicode; onError: @@ -1649,9 +2270,7 @@ PyUnicode_GET_SIZE(unicode), NULL); #elif defined(__APPLE__) - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - "surrogateescape"); + return _PyUnicode_AsUTF8String(unicode, "surrogateescape"); #else PyInterpreterState *interp = PyThreadState_GET()->interp; /* Bootstrap check: if the filesystem codec is implemented in Python, we @@ -1721,11 +2340,9 @@ if (encoding == NULL) { if (errors == NULL || strcmp(errors, "strict") == 0) - return PyUnicode_AsUTF8String(unicode); + return _PyUnicode_AsUTF8String(unicode, NULL); else - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return _PyUnicode_AsUTF8String(unicode, errors); } /* Shortcuts for common default encodings */ @@ -1734,18 +2351,14 @@ (strcmp(lower, "utf8") == 0)) { if (errors == NULL || strcmp(errors, "strict") == 0) - return PyUnicode_AsUTF8String(unicode); + return _PyUnicode_AsUTF8String(unicode, NULL); else - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return _PyUnicode_AsUTF8String(unicode, errors); } else if ((strcmp(lower, "latin-1") == 0) || (strcmp(lower, "latin1") == 0) || (strcmp(lower, "iso-8859-1") == 0)) - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return _PyUnicode_AsLatin1String(unicode, errors); #ifdef HAVE_MBCS else if (strcmp(lower, "mbcs") == 0) return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode), @@ -1753,9 +2366,7 @@ errors); #endif else if (strcmp(lower, "ascii") == 0) - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return _PyUnicode_AsASCIIString(unicode, errors); } /* Encode via the codec registry */ @@ -1824,21 +2435,6 @@ return NULL; } -PyObject * -_PyUnicode_AsDefaultEncodedString(PyObject *unicode) -{ - PyObject *v = ((PyUnicodeObject *)unicode)->defenc; - if (v) - return v; - v = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); - if (!v) - return NULL; - ((PyUnicodeObject *)unicode)->defenc = v; - return v; -} - PyObject* PyUnicode_DecodeFSDefault(const char *s) { Py_ssize_t size = (Py_ssize_t)strlen(s); @@ -1935,13 +2531,13 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) { PyObject *output = NULL; - Py_ssize_t size; - void *data; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); return 1; } if (PyUnicode_Check(arg)) { + if (PyUnicode_READY(arg)) + return 0; output = arg; Py_INCREF(output); } @@ -1960,9 +2556,8 @@ return 0; } } - size = PyUnicode_GET_SIZE(output); - data = PyUnicode_AS_UNICODE(output); - if (size != Py_UNICODE_strlen(data)) { + if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output), + PyUnicode_GET_LENGTH(output), 0, 1)) { PyErr_SetString(PyExc_TypeError, "embedded NUL character"); Py_DECREF(output); return 0; @@ -1973,53 +2568,219 @@ char* -_PyUnicode_AsStringAndSize(PyObject *unicode, Py_ssize_t *psize) +PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { PyObject *bytes; + PyUnicodeObject *u = (PyUnicodeObject *)unicode; + if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - bytes = _PyUnicode_AsDefaultEncodedString(unicode); - if (bytes == NULL) - return NULL; - if (psize != NULL) - *psize = PyBytes_GET_SIZE(bytes); - return PyBytes_AS_STRING(bytes); + if (PyUnicode_READY(u) == -1) + return NULL; + + if (_PyUnicode_UTF8(unicode) == NULL) { + bytes = _PyUnicode_AsUTF8String(unicode, "strict"); + if (bytes == NULL) + return NULL; + u->_base.utf8 = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); + if (u->_base.utf8 == NULL) { + Py_DECREF(bytes); + return NULL; + } + u->_base.utf8_length = PyBytes_GET_SIZE(bytes); + Py_MEMCPY(u->_base.utf8, PyBytes_AS_STRING(bytes), u->_base.utf8_length + 1); + Py_DECREF(bytes); + } + + if (psize) + *psize = _PyUnicode_UTF8_LENGTH(unicode); + return _PyUnicode_UTF8(unicode); } char* -_PyUnicode_AsString(PyObject *unicode) -{ - return _PyUnicode_AsStringAndSize(unicode, NULL); +PyUnicode_AsUTF8(PyObject *unicode) +{ + return PyUnicode_AsUTF8AndSize(unicode, NULL); +} + +#ifdef Py_DEBUG +int unicode_as_unicode_calls = 0; +#endif + + +Py_UNICODE * +PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) +{ + PyUnicodeObject *u; + const unsigned char *one_byte; +#if SIZEOF_WCHAR_T == 4 + const Py_UCS2 *two_bytes; +#else + const Py_UCS4 *four_bytes; + const Py_UCS4 *ucs4_end; + Py_ssize_t num_surrogates; +#endif + wchar_t *w; + wchar_t *wchar_end; + + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + u = (PyUnicodeObject*)unicode; + if (_PyUnicode_WSTR(u) == NULL) { + /* Non-ASCII compact unicode object */ + assert(_PyUnicode_KIND(u) != 0); + assert(PyUnicode_IS_READY(u)); + +#ifdef Py_DEBUG + ++unicode_as_unicode_calls; +#endif + + if (PyUnicode_KIND(u) == PyUnicode_4BYTE_KIND) { +#if SIZEOF_WCHAR_T == 2 + four_bytes = PyUnicode_4BYTE_DATA(u); + ucs4_end = four_bytes + _PyUnicode_LENGTH(u); + num_surrogates = 0; + + for (; four_bytes < ucs4_end; ++four_bytes) { + if (*four_bytes > 0xFFFF) + ++num_surrogates; + } + + _PyUnicode_WSTR(u) = (wchar_t *) PyObject_MALLOC( + sizeof(wchar_t) * (_PyUnicode_LENGTH(u) + 1 + num_surrogates)); + if (!_PyUnicode_WSTR(u)) { + PyErr_NoMemory(); + return NULL; + } + _PyUnicode_WSTR_LENGTH(u) = _PyUnicode_LENGTH(u) + num_surrogates; + + w = _PyUnicode_WSTR(u); + wchar_end = w + _PyUnicode_WSTR_LENGTH(u); + four_bytes = PyUnicode_4BYTE_DATA(u); + for (; four_bytes < ucs4_end; ++four_bytes, ++w) { + if (*four_bytes > 0xFFFF) { + /* encode surrogate pair in this case */ + *w++ = 0xD800 | ((*four_bytes - 0x10000) >> 10); + *w = 0xDC00 | ((*four_bytes - 0x10000) & 0x3FF); + } + else + *w = *four_bytes; + + if (w > wchar_end) { + assert(0 && "Miscalculated string end"); + } + } + *w = 0; +#else + /* sizeof(wchar_t) == 4 */ + Py_FatalError("Impossible unicode object state, wstr and str " + "should share memory already."); + return NULL; +#endif + } + else { + _PyUnicode_WSTR(u) = (wchar_t *) PyObject_MALLOC(sizeof(wchar_t) * + (_PyUnicode_LENGTH(u) + 1)); + if (!_PyUnicode_WSTR(u)) { + PyErr_NoMemory(); + return NULL; + } + if (!PyUnicode_IS_COMPACT_ASCII(u)) + _PyUnicode_WSTR_LENGTH(u) = _PyUnicode_LENGTH(u); + w = _PyUnicode_WSTR(u); + wchar_end = w + _PyUnicode_LENGTH(u); + + if (PyUnicode_KIND(u) == PyUnicode_1BYTE_KIND) { + one_byte = PyUnicode_1BYTE_DATA(u); + for (; w < wchar_end; ++one_byte, ++w) + *w = *one_byte; + /* null-terminate the wstr */ + *w = 0; + } + else if (PyUnicode_KIND(u) == PyUnicode_2BYTE_KIND) { +#if SIZEOF_WCHAR_T == 4 + two_bytes = PyUnicode_2BYTE_DATA(u); + for (; w < wchar_end; ++two_bytes, ++w) + *w = *two_bytes; + /* null-terminate the wstr */ + *w = 0; +#else + /* sizeof(wchar_t) == 2 */ + PyObject_FREE(_PyUnicode_WSTR(u)); + _PyUnicode_WSTR(u) = NULL; + Py_FatalError("Impossible unicode object state, wstr " + "and str should share memory already."); + return NULL; +#endif + } + else { + assert(0 && "This should never happen."); + } + } + } + if (size != NULL) + *size = PyUnicode_WSTR_LENGTH(u); + return _PyUnicode_WSTR(u); } Py_UNICODE * PyUnicode_AsUnicode(PyObject *unicode) { + return PyUnicode_AsUnicodeAndSize(unicode, NULL); +} + + +Py_ssize_t +PyUnicode_GetSize(PyObject *unicode) +{ if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); goto onError; } - return PyUnicode_AS_UNICODE(unicode); - - onError: - return NULL; -} - -Py_ssize_t -PyUnicode_GetSize(PyObject *unicode) -{ - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - goto onError; - } return PyUnicode_GET_SIZE(unicode); onError: return -1; } +Py_ssize_t +PyUnicode_GetLength(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode) || PyUnicode_READY(unicode) != -1) { + PyErr_BadArgument(); + return -1; + } + + return PyUnicode_GET_LENGTH(unicode); +} + +Py_UCS4 +PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) +{ + if (!PyUnicode_Check(unicode) || PyUnicode_READY(unicode) != -1) { + return PyErr_BadArgument(); + return (Py_UCS4)-1; + } + return PyUnicode_READ_CHAR(unicode, index); +} + +int +PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch) +{ + if (!PyUnicode_Check(unicode) || !PyUnicode_IS_COMPACT(unicode)) { + return PyErr_BadArgument(); + return -1; + } + + PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + index, ch); + return 0; +} + const char * PyUnicode_GetDefaultEncoding(void) { @@ -2075,7 +2836,7 @@ Py_ssize_t insize; Py_ssize_t requiredsize; Py_ssize_t newpos; - Py_UNICODE *repptr; + const Py_UNICODE *repptr; PyObject *inputobj = NULL; Py_ssize_t repsize; int res = -1; @@ -2281,7 +3042,7 @@ return (PyObject *)unicode; } - p = unicode->str; + p = PyUnicode_AS_UNICODE(unicode); shiftOutStart = p; e = s + size; @@ -2431,6 +3192,10 @@ Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); + return NULL; + } return (PyObject *)unicode; onError: @@ -2588,38 +3353,197 @@ # error C 'long' size should be either 4 or 8! #endif +/* Scans a UTF-8 string and returns the maximum character to be expected, + the size of the decoded unicode string and if any major errors were + encountered. + + This function does check basic UTF-8 sanity, it does however NOT CHECK + if the string contains surrogates, and if all continuation bytes are + within the correct ranges, these checks are performed in + PyUnicode_DecodeUTF8Stateful. + + If it sets has_errors to 1, it means the value of unicode_size and max_char + will be bogus and you should not rely on useful information in them. + */ +static Py_UCS4 +utf8_max_char_size_and_has_errors(const char *s, Py_ssize_t string_size, + Py_ssize_t *unicode_size, Py_ssize_t* consumed, + int *has_errors) +{ + Py_ssize_t n; + Py_ssize_t char_count = 0; + Py_UCS4 max_char = 127, new_max; + Py_UCS4 upper_bound; + const unsigned char *p = (const unsigned char *)s; + const unsigned char *end = p + string_size; + const unsigned char *aligned_end = (const unsigned char *) ((size_t) end & ~LONG_PTR_MASK); + int err = 0; + + for (; p < end && !err; ++p, ++char_count) { + /* Only check value if it's not a ASCII char... */ + if (*p < 0x80) { + /* Fast path, see below in PyUnicode_DecodeUTF8Stateful for + an explanation. */ + if (!((size_t) p & LONG_PTR_MASK)) { + /* Help register allocation */ + register const unsigned char *_p = p; + while (_p < aligned_end) { + unsigned long value = *(unsigned long *) _p; + if (value & ASCII_CHAR_MASK) + break; + _p += SIZEOF_LONG; + char_count += SIZEOF_LONG; + } + p = _p; + if (p == end) + break; + } + } + if (*p >= 0x80) { + n = utf8_code_length[*p]; + new_max = max_char; + switch (n) { + /* invalid start byte */ + case 0: + err = 1; + break; + case 2: + /* Code points between 0x00FF and 0x07FF inclusive. + Approximate the upper bound of the code point, + if this flips over 255 we can be sure it will be more + than 255 and the string will need 2 bytes per code coint, + if it stays under or equal to 255, we can be sure 1 byte + is enough. + ((*p & 0b00011111) << 6) | 0b00111111 */ + upper_bound = ((*p & 0x1F) << 6) | 0x3F; + if (max_char < upper_bound) + new_max = upper_bound; + /* Ensure we track at least that we left ASCII space. */ + if (new_max < 128) + new_max = 128; + break; + case 3: + /* Between 0x0FFF and 0xFFFF inclusive, so values are + always > 255 and <= 65535 and will always need 2 bytes. */ + if (max_char < 65535) + new_max = 65535; + break; + case 4: + /* Code point will be above 0xFFFF for sure in this case. */ + new_max = 65537; + break; + /* Internal error, this should be caught by the first if */ + case 1: + default: + assert(0 && "Impossible case in utf8_max_char_and_size"); + err = 1; + } + /* Instead of number of overall bytes for this code point, + n containts the number of following bytes: */ + --n; + /* Check if the follow up chars are all valid continuation bytes */ + if (n >= 1) { + const unsigned char *cont; + if ((p + n) >= end) { + if (consumed == 0) + /* incomplete data, non-incremental decoding */ + err = 1; + break; + } + for (cont = p + 1; cont < (p + n); ++cont) { + if ((*cont & 0xc0) != 0x80) { + err = 1; + break; + } + } + p += n; + } + else + err = 1; + max_char = new_max; + } + } + + if (unicode_size) + *unicode_size = char_count; + if (has_errors) + *has_errors = err; + return max_char; +} + +/* Similar to PyUnicode_WRITE but can also write into wstr field + of the legacy unicode representation */ +#define WRITE_FLEXIBLE_OR_WSTR(kind, buf, index, value) \ + do { \ + const int k_ = (kind); \ + if (k_ == PyUnicode_WCHAR_KIND) \ + ((Py_UNICODE *)(buf))[(index)] = (Py_UNICODE)(value); \ + else if (k_ == PyUnicode_1BYTE_KIND) \ + ((unsigned char *)(buf))[(index)] = (unsigned char)(value); \ + else if (k_ == PyUnicode_2BYTE_KIND) \ + ((Py_UCS2 *)(buf))[(index)] = (Py_UCS2)(value); \ + else \ + ((Py_UCS4 *)(buf))[(index)] = (Py_UCS4)(value); \ + } while (0) + PyObject * PyUnicode_DecodeUTF8Stateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { const char *starts = s; int n; int k; Py_ssize_t startinpos; Py_ssize_t endinpos; - Py_ssize_t outpos; const char *e, *aligned_end; PyUnicodeObject *unicode; - Py_UNICODE *p; const char *errmsg = ""; PyObject *errorHandler = NULL; PyObject *exc = NULL; - - /* Note: size will always be longer than the resulting Unicode - character count */ - unicode = _PyUnicode_New(size); - if (!unicode) - return NULL; + Py_UCS4 maxchar = 0; + Py_ssize_t unicode_size; + Py_ssize_t i; + int kind; + void *data; + int has_errors; + Py_UNICODE *error_outptr; +#if SIZEOF_WCHAR_T == 2 + Py_ssize_t wchar_offset = 0; +#endif + if (size == 0) { if (consumed) *consumed = 0; - return (PyObject *)unicode; - } - + return (PyObject *)PyUnicode_New(0, 0); + } + maxchar = utf8_max_char_size_and_has_errors(s, size, &unicode_size, + consumed, &has_errors); + if (has_errors) { + unicode = _PyUnicode_New(size); + if (!unicode) + return NULL; + kind = PyUnicode_WCHAR_KIND; + data = PyUnicode_AS_UNICODE(unicode); + assert(data != NULL); + } + else { + unicode = (PyUnicodeObject *)PyUnicode_New(unicode_size, maxchar); + if (!unicode) + return NULL; + /* When the string is ASCII only, just use memcpy and return. + unicode_size may be != size if there is an incomplete UTF-8 + sequence at the end of the ASCII block. */ + if (maxchar < 128 && size == unicode_size) { + Py_MEMCPY(PyUnicode_1BYTE_DATA(unicode), s, unicode_size); + return (PyObject *)unicode; + } + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + } /* Unpack UTF-8 encoded data */ - p = unicode->str; + i = 0; e = s + size; aligned_end = (const char *) ((size_t) e & ~LONG_PTR_MASK); @@ -2637,29 +3561,29 @@ if (!((size_t) s & LONG_PTR_MASK)) { /* Help register allocation */ register const char *_s = s; - register Py_UNICODE *_p = p; + register Py_ssize_t _i = i; while (_s < aligned_end) { /* Read a whole long at a time (either 4 or 8 bytes), and do a fast unrolled copy if it only contains ASCII characters. */ - unsigned long data = *(unsigned long *) _s; - if (data & ASCII_CHAR_MASK) + unsigned long value = *(unsigned long *) _s; + if (value & ASCII_CHAR_MASK) break; - _p[0] = (unsigned char) _s[0]; - _p[1] = (unsigned char) _s[1]; - _p[2] = (unsigned char) _s[2]; - _p[3] = (unsigned char) _s[3]; + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+0, _s[0]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+1, _s[1]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+2, _s[2]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+3, _s[3]); #if (SIZEOF_LONG == 8) - _p[4] = (unsigned char) _s[4]; - _p[5] = (unsigned char) _s[5]; - _p[6] = (unsigned char) _s[6]; - _p[7] = (unsigned char) _s[7]; + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+4, _s[4]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+5, _s[5]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+6, _s[6]); + WRITE_FLEXIBLE_OR_WSTR(kind, data, _i+7, _s[7]); #endif _s += SIZEOF_LONG; - _p += SIZEOF_LONG; + _i += SIZEOF_LONG; } s = _s; - p = _p; + i = _i; if (s == e) break; ch = (unsigned char)*s; @@ -2667,7 +3591,7 @@ } if (ch < 0x80) { - *p++ = (Py_UNICODE)ch; + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, ch); s++; continue; } @@ -2710,7 +3634,7 @@ } ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f); assert ((ch > 0x007F) && (ch <= 0x07FF)); - *p++ = (Py_UNICODE)ch; + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, ch); break; case 3: @@ -2739,7 +3663,7 @@ } ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); assert ((ch > 0x07FF) && (ch <= 0xFFFF)); - *p++ = (Py_UNICODE)ch; + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, ch); break; case 4: @@ -2764,19 +3688,27 @@ ((s[2] & 0x3f) << 6) + (s[3] & 0x3f); assert ((ch > 0xFFFF) && (ch <= 0x10ffff)); -#ifdef Py_UNICODE_WIDE - *p++ = (Py_UNICODE)ch; -#else - /* compute and append the two surrogates: */ - - /* translate from 10000..10FFFF to 0..FFFF */ - ch -= 0x10000; - - /* high surrogate = top 10 bits added to D800 */ - *p++ = (Py_UNICODE)(0xD800 + (ch >> 10)); - - /* low surrogate = bottom 10 bits added to DC00 */ - *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF)); + /* If the string is flexible or we have native UCS-4, write + directly.. */ + if (sizeof(Py_UNICODE) > 2 || kind != PyUnicode_WCHAR_KIND) + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, ch); + + else { + /* compute and append the two surrogates: */ + + /* translate from 10000..10FFFF to 0..FFFF */ + ch -= 0x10000; + + /* high surrogate = top 10 bits added to D800 */ + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, + (Py_UNICODE)(0xD800 + (ch >> 10))); + + /* low surrogate = bottom 10 bits added to DC00 */ + WRITE_FLEXIBLE_OR_WSTR(kind, data, i++, + (Py_UNICODE)(0xDC00 + (ch & 0x03FF))); + } +#if SIZEOF_WCHAR_T == 2 + wchar_offset++; #endif break; } @@ -2784,24 +3716,57 @@ continue; utf8Error: - outpos = p-PyUnicode_AS_UNICODE(unicode); + /* If this is not yet a resizable string, make it one.. */ + if (kind != PyUnicode_WCHAR_KIND) { + const Py_UNICODE *u; + PyUnicodeObject *new_unicode = _PyUnicode_New(size); + if (!new_unicode) + goto onError; + u = PyUnicode_AsUnicode((PyObject *)unicode); + if (!u) + goto onError; +#if SIZEOF_WCHAR_T == 2 + i += wchar_offset; +#endif + Py_UNICODE_COPY(PyUnicode_AS_UNICODE(new_unicode), u, i); + Py_DECREF(unicode); + unicode = new_unicode; + kind = 0; + data = PyUnicode_AS_UNICODE(new_unicode); + assert(data != NULL); + } + error_outptr = PyUnicode_AS_UNICODE(unicode) + i; if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf8", errmsg, &starts, &e, &startinpos, &endinpos, &exc, &s, - &unicode, &outpos, &p)) + &unicode, &i, &error_outptr)) goto onError; + /* Update data because unicode_decode_call_errorhandler might have + re-created or resized the unicode object. */ + data = PyUnicode_AS_UNICODE(unicode); aligned_end = (const char *) ((size_t) e & ~LONG_PTR_MASK); } + /* Ensure the unicode_size calculation above was correct: */ + assert(kind == PyUnicode_WCHAR_KIND || i == unicode_size); + if (consumed) *consumed = s-starts; - /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) - goto onError; + /* Adjust length and ready string when it contained errors and + is of the old resizable kind. */ + if (kind == PyUnicode_WCHAR_KIND) { + if (_PyUnicode_Resize(&unicode, i) < 0 || + PyUnicode_READY(unicode) == -1) + goto onError; + } Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); + return NULL; + } return (PyObject *)unicode; onError: @@ -2811,7 +3776,7 @@ return NULL; } -#undef ASCII_CHAR_MASK +#undef WRITE_FLEXIBLE_OR_WSTR #ifdef __APPLE__ @@ -2882,7 +3847,7 @@ } ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); assert ((ch > 0x07FF) && (ch <= 0xFFFF)); - *p++ = (Py_UNICODE)ch; + *p++ = (wchar_t)ch; break; case 4: @@ -2928,15 +3893,15 @@ #endif /* __APPLE__ */ -/* Allocation strategy: if the string is short, convert into a stack buffer +/* Primary internal function which creates utf8 encoded bytes objects. + + Allocation strategy: if the string is short, convert into a stack buffer and allocate exactly as much space needed at the end. Else allocate the maximum possible needed (4 result bytes per Unicode character), and return the excess memory at the end. */ PyObject * -PyUnicode_EncodeUTF8(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors) +_PyUnicode_AsUTF8String(PyObject *obj, const char *errors) { #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ @@ -2948,8 +3913,30 @@ char stackbuf[MAX_SHORT_UNICHARS * 4]; PyObject *errorHandler = NULL; PyObject *exc = NULL; - - assert(s != NULL); + int kind; + void *data; + Py_ssize_t size; + PyUnicodeObject *unicode = (PyUnicodeObject *)obj; +#if SIZEOF_WCHAR_T == 2 + Py_ssize_t wchar_offset = 0; +#endif + + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + + if (PyUnicode_READY(unicode) == -1) + return NULL; + + if (_PyUnicode_UTF8(unicode)) + return PyBytes_FromStringAndSize(_PyUnicode_UTF8(unicode), + _PyUnicode_UTF8_LENGTH(unicode)); + + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + size = PyUnicode_GET_LENGTH(unicode); + assert(size >= 0); if (size <= MAX_SHORT_UNICHARS) { @@ -2973,7 +3960,7 @@ } for (i = 0; i < size;) { - Py_UCS4 ch = s[i++]; + Py_UCS4 ch = PyUnicode_READ(kind, data, i++); if (ch < 0x80) /* Encode ASCII */ @@ -2984,83 +3971,72 @@ *p++ = (char)(0xc0 | (ch >> 6)); *p++ = (char)(0x80 | (ch & 0x3f)); } else if (0xD800 <= ch && ch <= 0xDFFF) { -#ifndef Py_UNICODE_WIDE - /* Special case: check for high and low surrogate */ - if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) { - Py_UCS4 ch2 = s[i]; - /* Combine the two surrogates to form a UCS4 value */ - ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; - i++; - - /* Encode UCS4 Unicode ordinals */ - *p++ = (char)(0xf0 | (ch >> 18)); - *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } else { -#endif - Py_ssize_t newpos; - PyObject *rep; - Py_ssize_t repsize, k; - rep = unicode_encode_call_errorhandler - (errors, &errorHandler, "utf-8", "surrogates not allowed", - s, size, &exc, i-1, i, &newpos); - if (!rep) + Py_ssize_t newpos; + PyObject *rep; + Py_ssize_t repsize, k, startpos; + startpos = i-1; +#if SIZEOF_WCHAR_T == 2 + startpos += wchar_offset; +#endif + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, "utf-8", "surrogates not allowed", + PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), + &exc, startpos, startpos+1, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_SIZE(rep); + + if (repsize > 4) { + Py_ssize_t offset; + + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { + /* integer overflow */ + PyErr_NoMemory(); goto error; - - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_SIZE(rep); - - if (repsize > 4) { - Py_ssize_t offset; - + } + nallocated += repsize - 4; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); - - if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { - /* integer overflow */ - PyErr_NoMemory(); + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; + } + + if (PyBytes_Check(rep)) { + char *prep = PyBytes_AS_STRING(rep); + for(k = repsize; k > 0; k--) + *p++ = *prep++; + } else /* rep is unicode */ { + const Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); + Py_UNICODE c; + + for(k=0; k<repsize; k++) { + c = prep[k]; + if (0x80 <= c) { + raise_encode_exception(&exc, "utf-8", + PyUnicode_AS_UNICODE(unicode), + size, i-1, i, + "surrogates not allowed"); goto error; } - nallocated += repsize - 4; - if (result != NULL) { - if (_PyBytes_Resize(&result, nallocated) < 0) - goto error; - } else { - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } - p = PyBytes_AS_STRING(result) + offset; + *p++ = (char)prep[k]; } - - if (PyBytes_Check(rep)) { - char *prep = PyBytes_AS_STRING(rep); - for(k = repsize; k > 0; k--) - *p++ = *prep++; - } else /* rep is unicode */ { - Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); - Py_UNICODE c; - - for(k=0; k<repsize; k++) { - c = prep[k]; - if (0x80 <= c) { - raise_encode_exception(&exc, "utf-8", s, size, - i-1, i, "surrogates not allowed"); - goto error; - } - *p++ = (char)prep[k]; - } - } - Py_DECREF(rep); -#ifndef Py_UNICODE_WIDE - } -#endif + } + Py_DECREF(rep); } else if (ch < 0x10000) { *p++ = (char)(0xe0 | (ch >> 12)); *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); @@ -3071,6 +4047,9 @@ *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); *p++ = (char)(0x80 | (ch & 0x3f)); +#if SIZEOF_WCHAR_T == 2 + wchar_offset++; +#endif } } @@ -3086,6 +4065,7 @@ assert(nneeded <= nallocated); _PyBytes_Resize(&result, nneeded); } + Py_XDECREF(errorHandler); Py_XDECREF(exc); return result; @@ -3099,18 +4079,24 @@ } PyObject * +PyUnicode_EncodeUTF8(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors) +{ + PyObject *v, *unicode; + + unicode = PyUnicode_FromUnicode(s, size); + if (unicode == NULL) + return NULL; + v = _PyUnicode_AsUTF8String(unicode, errors); + Py_DECREF(unicode); + return v; +} + +PyObject * PyUnicode_AsUTF8String(PyObject *unicode) { - PyObject *utf8; - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - utf8 = _PyUnicode_AsDefaultEncodedString(unicode); - if (utf8 == NULL) - return NULL; - Py_INCREF(utf8); - return utf8; + return _PyUnicode_AsUTF8String(unicode, NULL); } /* --- UTF-32 Codec ------------------------------------------------------- */ @@ -3222,7 +4208,7 @@ return (PyObject *)unicode; /* Unpack UTF-32 encoded data */ - p = unicode->str; + p = PyUnicode_AS_UNICODE(unicode); while (q < e) { Py_UCS4 ch; @@ -3275,11 +4261,15 @@ *consumed = (const char *)q-starts; /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); + return NULL; + } return (PyObject *)unicode; onError: @@ -3452,7 +4442,7 @@ return (PyObject *)unicode; /* Unpack UTF-16 encoded data */ - p = unicode->str; + p = PyUnicode_AS_UNICODE(unicode); q = (unsigned char *)s; e = q + size - 1; @@ -3669,11 +4659,15 @@ *consumed = (const char *)q-starts; /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); + return NULL; + } return (PyObject *)unicode; onError: @@ -3782,6 +4776,76 @@ /* --- Unicode Escape Codec ----------------------------------------------- */ +/* Helper function for PyUnicode_DecodeUnicodeEscape, determines + if all the escapes in the string make it still a valid ASCII string. + Returns -1 if any escapes were found which cause the string to + pop out of ASCII range. Otherwise returns the length of the + required buffer to hold the string. + */ +Py_ssize_t +length_of_escaped_ascii_string(const char *s, Py_ssize_t size) +{ + const unsigned char *p = (const unsigned char *)s; + const unsigned char *end = p + size; + Py_ssize_t length = 0; + + if (size < 0) + return -1; + + for (; p < end; ++p) { + if (*p > 127) { + /* Non-ASCII */ + return -1; + } + else if (*p != '\\') { + /* Normal character */ + ++length; + } + else { + /* Backslash-escape, check next char */ + ++p; + /* Escape sequence reaches till end of string or + non-ASCII follow-up. */ + if (p >= end || *p > 127) + return -1; + switch (*p) { + case '\n': + /* backslash + \n result in zero characters */ + break; + case '\\': case '\'': case '\"': + case 'b': case 'f': case 't': + case 'n': case 'r': case 'v': case 'a': + ++length; + break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + case 'x': case 'u': case 'U': case 'N': + /* these do not guarantee ASCII characters */ + return -1; + default: + /* count the backslash + the other character */ + length += 2; + } + } + } + return length; +} + +/* Similar to PyUnicode_WRITE but either write into wstr field + or treat string as ASCII. */ +#define WRITE_ASCII_OR_WSTR(kind, buf, index, value) \ + do { \ + if ((kind) != PyUnicode_WCHAR_KIND) \ + ((unsigned char *)(buf))[(index)] = (unsigned char)(value); \ + else \ + ((Py_UNICODE *)(buf))[(index)] = (Py_UNICODE)(value); \ + } while (0) + +#define WRITE_WSTR(buf, index, value) \ + assert(kind == PyUnicode_WCHAR_KIND), \ + ((Py_UNICODE *)(buf))[(index)] = (Py_UNICODE)(value) + + static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; PyObject * @@ -3792,8 +4856,7 @@ const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; - Py_ssize_t outpos; - int i; + int j; PyUnicodeObject *v; Py_UNICODE *p; const char *end; @@ -3801,19 +4864,42 @@ Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */ PyObject *errorHandler = NULL; PyObject *exc = NULL; - - /* Escaped strings will always be longer than the resulting - Unicode string, so we start with size here and then reduce the - length after conversion to the true value. - (but if the error callback returns a long replacement string - we'll have to allocate more space) */ - v = _PyUnicode_New(size); - if (v == NULL) - goto onError; + Py_ssize_t ascii_length; + Py_ssize_t i; + int kind; + void *data; + + ascii_length = length_of_escaped_ascii_string(s, size); + + /* After length_of_escaped_ascii_string() there are two alternatives, + either the string is pure ASCII with named escapes like \n, etc. + and we determined it's exact size (common case) + or it contains \x, \u, ... escape sequences. then we create a + legacy wchar string and resize it at the end of this function. */ + if (ascii_length >= 0) { + v = (PyUnicodeObject *)PyUnicode_New(ascii_length, 127); + if (!v) + goto onError; + assert(PyUnicode_KIND(v) == PyUnicode_1BYTE_KIND); + kind = PyUnicode_1BYTE_KIND; + data = PyUnicode_DATA(v); + } + else { + /* Escaped strings will always be longer than the resulting + Unicode string, so we start with size here and then reduce the + length after conversion to the true value. + (but if the error callback returns a long replacement string + we'll have to allocate more space) */ + v = _PyUnicode_New(size); + if (!v) + goto onError; + kind = PyUnicode_WCHAR_KIND; + data = PyUnicode_AS_UNICODE(v); + } + if (size == 0) return (PyObject *)v; - - p = PyUnicode_AS_UNICODE(v); + i = 0; end = s + size; while (s < end) { @@ -3821,9 +4907,18 @@ Py_UNICODE x; int digits; + if (kind == PyUnicode_WCHAR_KIND) { + assert(i < _PyUnicode_WSTR_LENGTH(v)); + } + else { + /* The only case in which i == ascii_length is a backslash + followed by a newline. */ + assert(i <= ascii_length); + } + /* Non-escape characters are interpreted as Unicode ordinals */ if (*s != '\\') { - *p++ = (unsigned char) *s++; + WRITE_ASCII_OR_WSTR(kind, data, i++, (unsigned char) *s++); continue; } @@ -3833,20 +4928,33 @@ c = *s++; if (s > end) c = '\0'; /* Invalid after \ */ + + if (kind == PyUnicode_WCHAR_KIND) { + assert(i < _PyUnicode_WSTR_LENGTH(v)); + } + else { + /* The only case in which i == ascii_length is a backslash + followed by a newline. */ + assert(i < ascii_length || (i == ascii_length && c == '\n')); + } + switch (c) { /* \x escapes */ case '\n': break; - case '\\': *p++ = '\\'; break; - case '\'': *p++ = '\''; break; - case '\"': *p++ = '\"'; break; - case 'b': *p++ = '\b'; break; - case 'f': *p++ = '\014'; break; /* FF */ - case 't': *p++ = '\t'; break; - case 'n': *p++ = '\n'; break; - case 'r': *p++ = '\r'; break; - case 'v': *p++ = '\013'; break; /* VT */ - case 'a': *p++ = '\007'; break; /* BEL, not classic C */ + case '\\': WRITE_ASCII_OR_WSTR(kind, data, i++, '\\'); break; + case '\'': WRITE_ASCII_OR_WSTR(kind, data, i++, '\''); break; + case '\"': WRITE_ASCII_OR_WSTR(kind, data, i++, '\"'); break; + case 'b': WRITE_ASCII_OR_WSTR(kind, data, i++, '\b'); break; + /* FF */ + case 'f': WRITE_ASCII_OR_WSTR(kind, data, i++, '\014'); break; + case 't': WRITE_ASCII_OR_WSTR(kind, data, i++, '\t'); break; + case 'n': WRITE_ASCII_OR_WSTR(kind, data, i++, '\n'); break; + case 'r': WRITE_ASCII_OR_WSTR(kind, data, i++, '\r'); break; + /* VT */ + case 'v': WRITE_ASCII_OR_WSTR(kind, data, i++, '\013'); break; + /* BEL, not classic C */ + case 'a': WRITE_ASCII_OR_WSTR(kind, data, i++, '\007'); break; /* \OOO (octal) escapes */ case '0': case '1': case '2': case '3': @@ -3857,7 +4965,7 @@ if (s < end && '0' <= *s && *s <= '7') x = (x<<3) + *s++ - '0'; } - *p++ = x; + WRITE_WSTR(data, i++, x); break; /* hex escapes */ @@ -3879,27 +4987,30 @@ message = "truncated \\UXXXXXXXX escape"; hexescape: chr = 0; - outpos = p-PyUnicode_AS_UNICODE(v); + p = PyUnicode_AS_UNICODE(v) + i; if (s+digits>end) { endinpos = size; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", "end of string in escape sequence", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i, &p)) goto onError; + data = PyUnicode_AS_UNICODE(v); goto nextByte; } - for (i = 0; i < digits; ++i) { - c = (unsigned char) s[i]; + for (j = 0; j < digits; ++j) { + c = (unsigned char) s[j]; if (!Py_ISXDIGIT(c)) { - endinpos = (s+i+1)-starts; + endinpos = (s+j+1)-starts; + p = PyUnicode_AS_UNICODE(v) + i; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i, &p)) goto onError; + data = PyUnicode_AS_UNICODE(v); goto nextByte; } chr = (chr<<4) & ~0xF; @@ -3910,7 +5021,7 @@ else chr += 10 + c - 'A'; } - s += i; + s += j; if (chr == 0xffffffff && PyErr_Occurred()) /* _decoding_error will have already written into the target buffer. */ @@ -3919,26 +5030,27 @@ /* when we get here, chr is a 32-bit unicode character */ if (chr <= 0xffff) /* UCS-2 character */ - *p++ = (Py_UNICODE) chr; + WRITE_WSTR(data, i++, chr); else if (chr <= 0x10ffff) { /* UCS-4 character. Either store directly, or as surrogate pair. */ #ifdef Py_UNICODE_WIDE - *p++ = chr; + WRITE_WSTR(data, i++, chr); #else chr -= 0x10000L; - *p++ = 0xD800 + (Py_UNICODE) (chr >> 10); - *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); + WRITE_WSTR(data, i++, 0xD800 + (Py_UNICODE) (chr >> 10)); + WRITE_WSTR(data, i++, 0xDC00 + (Py_UNICODE) (chr & 0x03FF)); #endif } else { endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); + p = PyUnicode_AS_UNICODE(v) + i; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", "illegal Unicode character", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i, &p)) goto onError; + data = PyUnicode_AS_UNICODE(v); } break; @@ -3947,7 +5059,8 @@ message = "malformed \\N character escape"; if (ucnhash_CAPI == NULL) { /* load the unicode data module */ - ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(PyUnicodeData_CAPSULE_NAME, 1); + ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import( + PyUnicodeData_CAPSULE_NAME, 1); if (ucnhash_CAPI == NULL) goto ucnhashError; } @@ -3960,43 +5073,51 @@ /* found a name. look it up in the unicode database */ message = "unknown Unicode character name"; s++; - if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr)) + if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), + &chr)) goto store; } } endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); + p = PyUnicode_AS_UNICODE(v) + i; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i, &p)) goto onError; + data = PyUnicode_AS_UNICODE(v); break; default: if (s > end) { + assert(kind == PyUnicode_WCHAR_KIND); message = "\\ at end of string"; s--; endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); + p = PyUnicode_AS_UNICODE(v) + i; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i, &p)) goto onError; + data = PyUnicode_AS_UNICODE(v); } else { - *p++ = '\\'; - *p++ = (unsigned char)s[-1]; + WRITE_ASCII_OR_WSTR(kind, data, i++, '\\'); + WRITE_ASCII_OR_WSTR(kind, data, i++, (unsigned char)s[-1]); } break; } nextByte: ; } - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + /* Ensure the length prediction worked in case of ASCII strings */ + assert(kind == PyUnicode_WCHAR_KIND || i == ascii_length); + + if (kind == PyUnicode_WCHAR_KIND && (_PyUnicode_Resize(&v, i) < 0 || + PyUnicode_READY(v) == -1)) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); @@ -4019,6 +5140,9 @@ return NULL; } +#undef WRITE_ASCII_OR_WSTR +#undef WRITE_WSTR + /* Return a Unicode-Escape string version of the Unicode object. If quotes is true, the string is enclosed in u"" or u'' quotes as @@ -4026,21 +5150,6 @@ */ -Py_LOCAL_INLINE(const Py_UNICODE *) findchar(const Py_UNICODE *s, - Py_ssize_t size, - Py_UNICODE ch) -{ - /* like wcschr, but doesn't stop at NULL characters */ - - while (size-- > 0) { - if (*s == ch) - return s; - s++; - } - - return NULL; -} - static const char *hexdigits = "0123456789abcdef"; PyObject * @@ -4309,6 +5418,10 @@ goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(v) == -1) { + Py_DECREF(v); + return NULL; + } return (PyObject *)v; onError: @@ -4447,7 +5560,9 @@ v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE); if (v == NULL) goto onError; - if (PyUnicode_GetSize((PyObject *)v) == 0) + /* Intentionally PyUnicode_GET_SIZE instead of PyUnicode_GET_LENGTH + as string was created with the old API. */ + if (PyUnicode_GET_SIZE(v) == 0) return (PyObject *)v; p = PyUnicode_AS_UNICODE(v); end = s + size; @@ -4491,6 +5606,10 @@ goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(v) == -1) { + Py_DECREF(v); + return NULL; + } return (PyObject *)v; onError: @@ -4507,41 +5626,8 @@ Py_ssize_t size, const char *errors) { - PyUnicodeObject *v; - Py_UNICODE *p; - const char *e, *unrolled_end; - /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ - if (size == 1) { - Py_UNICODE r = *(unsigned char*)s; - return PyUnicode_FromUnicode(&r, 1); - } - - v = _PyUnicode_New(size); - if (v == NULL) - goto onError; - if (size == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); - e = s + size; - /* Unrolling the copy makes it much faster by reducing the looping - overhead. This is similar to what many memcpy() implementations do. */ - unrolled_end = e - 4; - while (s < unrolled_end) { - p[0] = (unsigned char) s[0]; - p[1] = (unsigned char) s[1]; - p[2] = (unsigned char) s[2]; - p[3] = (unsigned char) s[3]; - s += 4; - p += 4; - } - while (s < e) - *p++ = (unsigned char) *s++; - return (PyObject *)v; - - onError: - Py_XDECREF(v); - return NULL; + return PyUnicode_FromUCS1((unsigned char*)s, size); } /* create or adjust a UnicodeEncodeError */ @@ -4849,15 +5935,30 @@ } PyObject * -PyUnicode_AsLatin1String(PyObject *unicode) +_PyUnicode_AsLatin1String(PyObject *unicode, const char *errors) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } + if (PyUnicode_READY(unicode) == -1) + return NULL; + /* Fast path: if it is a one-byte string, construct + bytes object directly. */ + if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) + return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), + PyUnicode_GET_LENGTH(unicode)); + /* Non-Latin-1 characters present. Defer to above function to + raise the exception. */ return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), - NULL); + errors); +} + +PyObject* +PyUnicode_AsLatin1String(PyObject *unicode) +{ + return _PyUnicode_AsLatin1String(unicode, NULL); } /* --- 7-bit ASCII Codec -------------------------------------------------- */ @@ -4874,14 +5975,32 @@ Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; + unsigned char* d; PyObject *errorHandler = NULL; PyObject *exc = NULL; + Py_ssize_t i; /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && *(unsigned char*)s < 128) { - Py_UNICODE r = *(unsigned char*)s; - return PyUnicode_FromUnicode(&r, 1); - } + if (size == 1 && *(unsigned char*)s < 128) + return PyUnicode_FromOrdinal(*(unsigned char*)s); + + /* Fast path. Assume the input actually *is* ASCII, and allocate + a single-block Unicode object with that assumption. If there is + an error, drop the object and start over. */ + v = (PyUnicodeObject*)PyUnicode_New(size, 127); + if (v == NULL) + goto onError; + d = PyUnicode_1BYTE_DATA(v); + for (i = 0; i < size; i++) { + unsigned char ch = ((unsigned char*)s)[i]; + if (ch < 128) + d[i] = ch; + else + break; + } + if (i == size) + return (PyObject*)v; + Py_DECREF(v); /* start over */ v = _PyUnicode_New(size); if (v == NULL) @@ -4913,6 +6032,10 @@ goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(v) == -1) { + Py_DECREF(v); + return NULL; + } return (PyObject *)v; onError: @@ -4931,15 +6054,28 @@ } PyObject * -PyUnicode_AsASCIIString(PyObject *unicode) +_PyUnicode_AsASCIIString(PyObject *unicode, const char *errors) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } + if (PyUnicode_READY(unicode) == -1) + return NULL; + /* Fast path: if it is an ASCII-only string, construct bytes object + directly. Else defer to above function to raise the exception. */ + if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128) + return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), + PyUnicode_GET_LENGTH(unicode)); return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), - NULL); + errors); +} + +PyObject * +PyUnicode_AsASCIIString(PyObject *unicode) +{ + return _PyUnicode_AsASCIIString(unicode, NULL); } #ifdef HAVE_MBCS @@ -5090,7 +6226,10 @@ goto retry; } #endif - + if (PyUnicode_READY(v) == -1) { + Py_DECREF(v); + return NULL; + } return (PyObject *)v; } @@ -5386,6 +6525,10 @@ goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); + if (PyUnicode_READY(v) == -1) { + Py_DECREF(v); + return NULL; + } return (PyObject *)v; onError: @@ -5471,7 +6614,6 @@ PyObject* PyUnicode_BuildEncodingMap(PyObject* string) { - Py_UNICODE *decode; PyObject *result; struct encoding_map *mresult; int i; @@ -5480,35 +6622,36 @@ unsigned char level2[512]; unsigned char *mlevel1, *mlevel2, *mlevel3; int count2 = 0, count3 = 0; - - if (!PyUnicode_Check(string) || PyUnicode_GetSize(string) != 256) { + int kind; + void *data; + Py_UCS4 ch; + + if (!PyUnicode_Check(string) || PyUnicode_GET_LENGTH(string) != 256) { PyErr_BadArgument(); return NULL; } - decode = PyUnicode_AS_UNICODE(string); + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); /* If there isn't a one-to-one mapping of NULL to \0, or if there are non-BMP characters, we need to use a mapping dictionary. */ - if (decode[0] != 0) + if (PyUnicode_READ(kind, data, 0) != 0) need_dict = 1; for (i = 1; i < 256; i++) { int l1, l2; - if (decode[i] == 0 -#ifdef Py_UNICODE_WIDE - || decode[i] > 0xFFFF -#endif - ) { + ch = PyUnicode_READ(kind, data, i); + if (ch == 0 || ch > 0xFFFF) { need_dict = 1; break; } - if (decode[i] == 0xFFFE) + if (ch == 0xFFFE) /* unmapped character */ continue; - l1 = decode[i] >> 11; - l2 = decode[i] >> 7; + l1 = ch >> 11; + l2 = ch >> 7; if (level1[l1] == 0xFF) level1[l1] = count2++; if (level2[l2] == 0xFF) @@ -5524,7 +6667,7 @@ if (!result) return NULL; for (i = 0; i < 256; i++) { - key = PyLong_FromLong(decode[i]); + key = PyLong_FromLong(PyUnicode_READ(kind, data, i)); value = PyLong_FromLong(i); if (!key || !value) goto failed1; @@ -5559,15 +6702,15 @@ count3 = 0; for (i = 1; i < 256; i++) { int o1, o2, o3, i2, i3; - if (decode[i] == 0xFFFE) + if (PyUnicode_READ(kind, data, i) == 0xFFFE) /* unmapped character */ continue; - o1 = decode[i]>>11; - o2 = (decode[i]>>7) & 0xF; + o1 = PyUnicode_READ(kind, data, i)>>11; + o2 = (PyUnicode_READ(kind, data, i)>>7) & 0xF; i2 = 16*mlevel1[o1] + o2; if (mlevel2[i2] == 0xFF) mlevel2[i2] = count3++; - o3 = decode[i] & 0x7F; + o3 = PyUnicode_READ(kind, data, i) & 0x7F; i3 = 128*mlevel2[i2] + o3; mlevel3[i3] = i; } @@ -5951,13 +7094,13 @@ /* create or adjust a UnicodeTranslateError */ static void make_translate_exception(PyObject **exceptionObject, - const Py_UNICODE *unicode, Py_ssize_t size, + PyObject *unicode, Py_ssize_t startpos, Py_ssize_t endpos, const char *reason) { if (*exceptionObject == NULL) { - *exceptionObject = PyUnicodeTranslateError_Create( - unicode, size, startpos, endpos, reason); + *exceptionObject = _PyUnicodeTranslateError_Create( + unicode, startpos, endpos, reason); } else { if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos)) @@ -5976,12 +7119,12 @@ /* raises a UnicodeTranslateError */ static void raise_translate_exception(PyObject **exceptionObject, - const Py_UNICODE *unicode, Py_ssize_t size, + PyObject *unicode, Py_ssize_t startpos, Py_ssize_t endpos, const char *reason) { make_translate_exception(exceptionObject, - unicode, size, startpos, endpos, reason); + unicode, startpos, endpos, reason); if (*exceptionObject != NULL) PyCodec_StrictErrors(*exceptionObject); } @@ -5994,7 +7137,7 @@ unicode_translate_call_errorhandler(const char *errors, PyObject **errorHandler, const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + PyObject *unicode, PyObject **exceptionObject, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { @@ -6011,7 +7154,7 @@ } make_translate_exception(exceptionObject, - unicode, size, startpos, endpos, reason); + unicode, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; @@ -6030,10 +7173,10 @@ return NULL; } if (i_newpos<0) - *newpos = size+i_newpos; + *newpos = PyUnicode_GET_LENGTH(unicode)+i_newpos; else *newpos = i_newpos; - if (*newpos<0 || *newpos>size) { + if (*newpos<0 || *newpos>PyUnicode_GET_LENGTH(unicode)) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -6047,7 +7190,7 @@ which must be decrefed by the caller. Return 0 on success, -1 on error */ static int -charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result) +charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; @@ -6097,19 +7240,18 @@ if not reallocate and adjust various state variables. Return 0 on success, -1 on error */ static int -charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp, +charmaptranslate_makespace(Py_UCS4 **outobj, Py_ssize_t *psize, Py_ssize_t requiredsize) { - Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj); + Py_ssize_t oldsize = *psize; if (requiredsize > oldsize) { - /* remember old output position */ - Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj); /* exponentially overallocate to minimize reallocations */ if (requiredsize < 2 * oldsize) requiredsize = 2 * oldsize; - if (PyUnicode_Resize(outobj, requiredsize) < 0) + *outobj = PyMem_Realloc(*outobj, requiredsize * sizeof(Py_UCS4)); + if (*outobj == 0) return -1; - *outp = PyUnicode_AS_UNICODE(*outobj) + outpos; + *psize = requiredsize; } return 0; } @@ -6120,37 +7262,43 @@ The called must decref result. Return 0 on success, -1 on error. */ static int -charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp, - Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp, +charmaptranslate_output(PyObject *input, Py_ssize_t ipos, + PyObject *mapping, Py_UCS4 **output, + Py_ssize_t *osize, Py_ssize_t *opos, PyObject **res) { - if (charmaptranslate_lookup(*curinp, mapping, res)) + Py_UCS4 curinp = PyUnicode_READ_CHAR(input, ipos); + if (charmaptranslate_lookup(curinp, mapping, res)) return -1; if (*res==NULL) { /* not found => default to 1:1 mapping */ - *(*outp)++ = *curinp; + (*output)[(*opos)++] = curinp; } else if (*res==Py_None) ; else if (PyLong_Check(*res)) { /* no overflow check, because we know that the space is enough */ - *(*outp)++ = (Py_UNICODE)PyLong_AS_LONG(*res); + (*output)[(*opos)++] = (Py_UCS4)PyLong_AS_LONG(*res); } else if (PyUnicode_Check(*res)) { - Py_ssize_t repsize = PyUnicode_GET_SIZE(*res); + Py_ssize_t repsize; + if (PyUnicode_READY(*res) == -1) + return -1; + repsize = PyUnicode_GET_LENGTH(*res); if (repsize==1) { /* no overflow check, because we know that the space is enough */ - *(*outp)++ = *PyUnicode_AS_UNICODE(*res); + (*output)[(*opos)++] = PyUnicode_READ_CHAR(*res, 0); } else if (repsize!=0) { /* more than one character */ - Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) + - (insize - (curinp-startinp)) + + Py_ssize_t requiredsize = *opos + + (PyUnicode_GET_LENGTH(input) - ipos) + repsize - 1; - if (charmaptranslate_makespace(outobj, outp, requiredsize)) + Py_ssize_t i; + if (charmaptranslate_makespace(output, osize, requiredsize)) return -1; - memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize); - *outp += repsize; + for(i = 0; i < repsize; i++) + (*output)[(*opos)++] = PyUnicode_READ_CHAR(*res, i); } } else @@ -6159,20 +7307,20 @@ } PyObject * -PyUnicode_TranslateCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) -{ - /* output object */ - PyObject *res = NULL; - /* pointers to the beginning and end+1 of input */ - const Py_UNICODE *startp = p; - const Py_UNICODE *endp = p + size; - /* pointer into the output */ - Py_UNICODE *str; +_PyUnicode_TranslateCharmap(PyObject *input, + PyObject *mapping, + const char *errors) +{ + /* input object */ + char *idata; + Py_ssize_t size, i; + int kind; + /* output buffer */ + Py_UCS4 *output = NULL; + Py_ssize_t osize; + PyObject *res; /* current output position */ - Py_ssize_t respos = 0; + Py_ssize_t opos; char *reason = "character maps to <undefined>"; PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -6186,38 +7334,52 @@ return NULL; } + if (PyUnicode_READY(input) == -1) + return NULL; + idata = (char*)PyUnicode_DATA(input); + kind = PyUnicode_KIND(input); + size = PyUnicode_GET_LENGTH(input); + i = 0; + + if (size == 0) { + Py_INCREF(input); + return input; + } + /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ - res = PyUnicode_FromUnicode(NULL, size); - if (res == NULL) + osize = size; + output = PyMem_Malloc(osize * sizeof(Py_UCS4)); + opos = 0; + if (output == NULL) { + PyErr_NoMemory(); goto onError; - if (size == 0) - return res; - str = PyUnicode_AS_UNICODE(res); - - while (p<endp) { + } + + while (i<size) { /* try to encode it */ PyObject *x = NULL; - if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) { + if (charmaptranslate_output(input, i, mapping, + &output, &osize, &opos, &x)) { Py_XDECREF(x); goto onError; } Py_XDECREF(x); if (x!=Py_None) /* it worked => adjust input pointer */ - ++p; + ++i; else { /* untranslatable character */ PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ Py_ssize_t repsize; Py_ssize_t newpos; - Py_UNICODE *uni2; + Py_ssize_t uni2; /* startpos for collecting untranslatable chars */ - const Py_UNICODE *collstart = p; - const Py_UNICODE *collend = p+1; - const Py_UNICODE *coll; + Py_ssize_t collstart = i; + Py_ssize_t collend = i+1; + Py_ssize_t coll; /* find all untranslatable characters */ - while (collend < endp) { - if (charmaptranslate_lookup(*collend, mapping, &x)) + while (collend < size) { + if (charmaptranslate_lookup(PyUnicode_READ(kind,idata, collend), mapping, &x)) goto onError; Py_XDECREF(x); if (x!=Py_None) @@ -6240,67 +7402,79 @@ } switch (known_errorHandler) { case 1: /* strict */ - raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason); + raise_translate_exception(&exc, input, collstart, + collend, reason); goto onError; case 2: /* replace */ /* No need to check for space, this is a 1:1 replacement */ - for (coll = collstart; coll<collend; ++coll) - *str++ = '?'; + for (coll = collstart; coll<collend; coll++) + output[opos++] = '?'; /* fall through */ case 3: /* ignore */ - p = collend; + i = collend; break; case 4: /* xmlcharrefreplace */ - /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { + /* generate replacement (temporarily (mis)uses i) */ + for (i = collstart; i < collend; ++i) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)*p); - if (charmaptranslate_makespace(&res, &str, - (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend))) + sprintf(buffer, "&#%d;", PyUnicode_READ(kind, idata, i)); + if (charmaptranslate_makespace(&output, &osize, + opos+strlen(buffer)+(size-collend))) goto onError; for (cp = buffer; *cp; ++cp) - *str++ = *cp; + output[opos++] = *cp; } - p = collend; + i = collend; break; default: repunicode = unicode_translate_call_errorhandler(errors, &errorHandler, - reason, startp, size, &exc, - collstart-startp, collend-startp, &newpos); - if (repunicode == NULL) + reason, input, &exc, + collstart, collend, &newpos); + if (repunicode == NULL || PyUnicode_READY(repunicode) == -1) goto onError; /* generate replacement */ - repsize = PyUnicode_GET_SIZE(repunicode); - if (charmaptranslate_makespace(&res, &str, - (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) { + repsize = PyUnicode_GET_LENGTH(repunicode); + if (charmaptranslate_makespace(&output, &osize, + opos+repsize+(size-collend))) { Py_DECREF(repunicode); goto onError; } - for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) - *str++ = *uni2; - p = startp + newpos; + for (uni2 = 0; repsize-->0; ++uni2) + output[opos++] = PyUnicode_READ_CHAR(repunicode, uni2); + i = newpos; Py_DECREF(repunicode); } } } - /* Resize if we allocated to much */ - respos = str-PyUnicode_AS_UNICODE(res); - if (respos<PyUnicode_GET_SIZE(res)) { - if (PyUnicode_Resize(&res, respos) < 0) - goto onError; - } + res = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, opos); + if (!res) + goto onError; + PyMem_Free(output); Py_XDECREF(exc); Py_XDECREF(errorHandler); return res; onError: - Py_XDECREF(res); + PyMem_Free(output); Py_XDECREF(exc); Py_XDECREF(errorHandler); return NULL; } +/* Deprecated. Use PyUnicode_Translate instead. */ +PyObject * +PyUnicode_TranslateCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) +{ + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (!unicode) + return NULL; + return _PyUnicode_TranslateCharmap(unicode, mapping, errors); +} + PyObject * PyUnicode_Translate(PyObject *str, PyObject *mapping, @@ -6311,10 +7485,7 @@ str = PyUnicode_FromObject(str); if (str == NULL) goto onError; - result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str), - PyUnicode_GET_SIZE(str), - mapping, - errors); + result = _PyUnicode_TranslateCharmap(str, mapping, errors); Py_DECREF(str); return result; @@ -6323,6 +7494,60 @@ return NULL; } +static Py_UCS4 +fix_decimal_and_space_to_ascii(PyUnicodeObject *self) +{ + /* No need to call PyUnicode_READY(self) because this function is only + called as a callback from fixup() which does it already. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + Py_UCS4 maxchar = 0, ch, fixed; + Py_ssize_t i; + + for (i = 0; i < len; ++i) { + ch = PyUnicode_READ(kind, data, i); + fixed = 0; + if (ch > 127) { + if (Py_UNICODE_ISSPACE(ch)) + fixed = ' '; + else { + const int decimal = Py_UNICODE_TODECIMAL(ch); + if (decimal >= 0) + fixed = '0' + decimal; + } + if (fixed != 0) { + if (fixed > maxchar) + maxchar = fixed; + PyUnicode_WRITE(kind, data, i, fixed); + } + else if (ch > maxchar) + maxchar = ch; + } + else if (ch > maxchar) + maxchar = ch; + } + + return maxchar; +} + +PyObject * +_PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadInternalCall(); + return NULL; + } + if (PyUnicode_READY(unicode) == -1) + return NULL; + if (PyUnicode_MAX_CHAR_VALUE(unicode) <= 127) { + /* If the string is already ASCII, just return the same string */ + Py_INCREF(unicode); + return unicode; + } + return fixup((PyUnicodeObject *)unicode, fix_decimal_and_space_to_ascii); +} + PyObject * PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t length) @@ -6345,6 +7570,10 @@ p[i] = '0' + decimal; } } + if (PyUnicode_READY((PyUnicodeObject*)result) == -1) { + Py_DECREF(result); + return NULL; + } return result; } /* --- Decimal Encoder ---------------------------------------------------- */ @@ -6487,17 +7716,123 @@ /* --- Helpers ------------------------------------------------------------ */ +#include "stringlib/ucs1lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs2lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs4lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +static Py_ssize_t +any_find_slice(Py_ssize_t Py_LOCAL_CALLBACK(ucs1)(const Py_UCS1*, Py_ssize_t, + const Py_UCS1*, Py_ssize_t, + Py_ssize_t, Py_ssize_t), + Py_ssize_t Py_LOCAL_CALLBACK(ucs2)(const Py_UCS2*, Py_ssize_t, + const Py_UCS2*, Py_ssize_t, + Py_ssize_t, Py_ssize_t), + Py_ssize_t Py_LOCAL_CALLBACK(ucs4)(const Py_UCS4*, Py_ssize_t, + const Py_UCS4*, Py_ssize_t, + Py_ssize_t, Py_ssize_t), + PyObject* s1, PyObject* s2, + Py_ssize_t start, + Py_ssize_t end) +{ + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2, result; + + kind1 = PyUnicode_KIND(s1); + kind2 = PyUnicode_KIND(s2); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(s1); + buf2 = PyUnicode_DATA(s2); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(s1, kind); + if (!buf1) + return -2; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(s2, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return -2; + } + len1 = PyUnicode_GET_LENGTH(s1); + len2 = PyUnicode_GET_LENGTH(s2); + + switch(kind) { + case PyUnicode_1BYTE_KIND: + result = ucs1(buf1, len1, buf2, len2, start, end); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2(buf1, len1, buf2, len2, start, end); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4(buf1, len1, buf2, len2, start, end); + break; + default: + assert(0); result = -2; + } + + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + + return result; +} + +Py_ssize_t +_PyUnicode_InsertThousandsGrouping(int kind, void *data, + Py_ssize_t n_buffer, + void *digits, Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + const char *thousands_sep) +{ + switch(kind) { + case PyUnicode_1BYTE_KIND: + return _PyUnicode_ucs1_InsertThousandsGrouping( + (Py_UCS1*)data, n_buffer, (Py_UCS1*)digits, n_digits, + min_width, grouping, thousands_sep); + case PyUnicode_2BYTE_KIND: + return _PyUnicode_ucs2_InsertThousandsGrouping( + (Py_UCS2*)data, n_buffer, (Py_UCS2*)digits, n_digits, + min_width, grouping, thousands_sep); + case PyUnicode_4BYTE_KIND: + return _PyUnicode_ucs4_InsertThousandsGrouping( + (Py_UCS4*)data, n_buffer, (Py_UCS4*)digits, n_digits, + min_width, grouping, thousands_sep); + } + assert(0); + return -1; +} + + #include "stringlib/unicodedefs.h" #include "stringlib/fastsearch.h" #include "stringlib/count.h" #include "stringlib/find.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" - -#define _Py_InsertThousandsGrouping _PyUnicode_InsertThousandsGrouping -#define _Py_InsertThousandsGroupingLocale _PyUnicode_InsertThousandsGroupingLocale -#include "stringlib/localeutil.h" /* helper macro to fixup start/end slice values */ #define ADJUST_INDICES(start, end, len) \ @@ -6514,28 +7849,6 @@ start = 0; \ } -/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed - * by 'ptr', possibly combining surrogate pairs on narrow builds. - * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character - * that should be returned and 'end' pointing to the end of the buffer. - * ('end' is used on narrow builds to detect a lone surrogate at the - * end of the buffer that should be returned unchanged.) - * The ptr and end arguments should be side-effect free and ptr must an lvalue. - * The type of the returned char is always Py_UCS4. - * - * Note: the macro advances ptr to next char, so it might have side-effects - * (especially if used with other macros). - */ -#ifdef Py_UNICODE_WIDE -#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++ -#else -#define _Py_UNICODE_NEXT(ptr, end) \ - (((Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \ - Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \ - ((ptr) += 2,Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \ - (Py_UCS4)*(ptr)++) -#endif - Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, @@ -6545,26 +7858,76 @@ Py_ssize_t result; PyUnicodeObject* str_obj; PyUnicodeObject* sub_obj; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str); - if (!str_obj) + if (!str_obj || PyUnicode_READY(str_obj) == -1) return -1; sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr); - if (!sub_obj) { + if (!sub_obj || PyUnicode_READY(str_obj) == -1) { Py_DECREF(str_obj); return -1; } - ADJUST_INDICES(start, end, str_obj->length); - result = stringlib_count( - str_obj->str + start, end - start, sub_obj->str, sub_obj->length, - PY_SSIZE_T_MAX - ); + kind1 = PyUnicode_KIND(str_obj); + kind2 = PyUnicode_KIND(sub_obj); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(str_obj); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind((PyObject*)str_obj, kind); + if (!buf1) + goto onError; + buf2 = PyUnicode_DATA(sub_obj); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind((PyObject*)sub_obj, kind); + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sub_obj); + + ADJUST_INDICES(start, end, len1); + switch(kind) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_count( + ((Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + assert(0); result = 0; + } Py_DECREF(sub_obj); Py_DECREF(str_obj); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return result; + onError: + Py_DECREF(sub_obj); + Py_DECREF(str_obj); + if (kind1 != kind && buf1) + PyMem_Free(buf1); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return -1; } Py_ssize_t @@ -6577,25 +7940,23 @@ Py_ssize_t result; str = PyUnicode_FromObject(str); - if (!str) + if (!str || PyUnicode_READY(str) == -1) return -2; sub = PyUnicode_FromObject(sub); - if (!sub) { + if (!sub || PyUnicode_READY(sub) == -1) { Py_DECREF(str); return -2; } if (direction > 0) - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), - PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), - start, end + result = any_find_slice( + ucs1lib_find_slice, ucs2lib_find_slice, ucs4lib_find_slice, + str, sub, start, end ); else - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), - PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), - start, end + result = any_find_slice( + ucs1lib_rfind_slice, ucs2lib_rfind_slice, ucs4lib_rfind_slice, + str, sub, start, end ); Py_DECREF(str); @@ -6604,6 +7965,27 @@ return result; } +Py_ssize_t +PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, + Py_ssize_t start, Py_ssize_t end, + int direction) +{ + char *result; + int kind; + if (PyUnicode_READY(str) == -1) + return -2; + if (end > PyUnicode_GET_LENGTH(str)) + end = PyUnicode_GET_LENGTH(str); + kind = PyUnicode_KIND(str); + result = findchar(PyUnicode_1BYTE_DATA(str) + + PyUnicode_KIND_SIZE(kind, start), + kind, + end-start, ch, direction); + if (!result) + return -1; + return (result-(char*)PyUnicode_DATA(str)) >> (kind-1); +} + static int tailmatch(PyUnicodeObject *self, PyUnicodeObject *substring, @@ -6611,20 +7993,62 @@ Py_ssize_t end, int direction) { - if (substring->length == 0) + int kind_self; + int kind_sub; + void *data_self; + void *data_sub; + Py_ssize_t offset; + Py_ssize_t i; + Py_ssize_t end_sub; + + if (PyUnicode_READY(self) == -1 || + PyUnicode_READY(substring) == -1) + return 0; + + if (PyUnicode_GET_LENGTH(substring) == 0) return 1; - ADJUST_INDICES(start, end, self->length); - end -= substring->length; + ADJUST_INDICES(start, end, PyUnicode_GET_LENGTH(self)); + end -= PyUnicode_GET_LENGTH(substring); if (end < start) return 0; - if (direction > 0) { - if (Py_UNICODE_MATCH(self, end, substring)) + kind_self = PyUnicode_KIND(self); + data_self = PyUnicode_DATA(self); + kind_sub = PyUnicode_KIND(substring); + data_sub = PyUnicode_DATA(substring); + end_sub = PyUnicode_GET_LENGTH(substring) - 1; + + if (direction > 0) + offset = end; + else + offset = start; + + if (PyUnicode_READ(kind_self, data_self, offset) == + PyUnicode_READ(kind_sub, data_sub, 0) && + PyUnicode_READ(kind_self, data_self, offset + end_sub) == + PyUnicode_READ(kind_sub, data_sub, end_sub)) { + /* If both are of the same kind, memcmp is sufficient */ + if (kind_self == kind_sub) { + return ! memcmp((char *)data_self + + (offset * PyUnicode_CHARACTER_SIZE(substring)), + data_sub, + PyUnicode_GET_LENGTH(substring) * + PyUnicode_CHARACTER_SIZE(substring)); + } + /* otherwise we have to compare each character by first accesing it */ + else { + /* We do not need to compare 0 and len(substring)-1 because + the if statement above ensured already that they are equal + when we end up here. */ + // TODO: honor direction and do a forward or backwards search + for (i = 1; i < end_sub; ++i) { + if (PyUnicode_READ(kind_self, data_self, offset + i) != + PyUnicode_READ(kind_sub, data_sub, i)) + return 0; + } return 1; - } else { - if (Py_UNICODE_MATCH(self, start, substring)) - return 1; + } } return 0; @@ -6661,18 +8085,39 @@ static PyObject * fixup(PyUnicodeObject *self, - int (*fixfct)(PyUnicodeObject *s)) -{ - - PyUnicodeObject *u; - - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + Py_UCS4 (*fixfct)(PyUnicodeObject *s)) +{ + PyObject *u; + Py_UCS4 maxchar_old, maxchar_new = 0; + + if (PyUnicode_READY(self) == -1) + return NULL; + maxchar_old = PyUnicode_MAX_CHAR_VALUE(self); + u = PyUnicode_New(PyUnicode_GET_LENGTH(self), + maxchar_old); if (u == NULL) return NULL; - Py_UNICODE_COPY(u->str, self->str, self->length); - - if (!fixfct(u) && PyUnicode_CheckExact(self)) { + Py_MEMCPY(PyUnicode_1BYTE_DATA(u), PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(u) * PyUnicode_CHARACTER_SIZE(u)); + + /* fix functions return the new maximum character in a string, + if the kind of the resulting unicode object does not change, + everything is fine. Otherwise we need to change the string kind + and re-run the fix function. */ + maxchar_new = fixfct((PyUnicodeObject*)u); + if (maxchar_new == 0) + /* do nothing, keep maxchar_new at 0 which means no changes. */; + else if (maxchar_new <= 127) + maxchar_new = 127; + else if (maxchar_new <= 255) + maxchar_new = 255; + else if (maxchar_new <= 65535) + maxchar_new = 65535; + else + maxchar_new = 1114111; /* 0x10ffff */ + + if (!maxchar_new && PyUnicode_CheckExact(self)) { /* fixfct should return TRUE if it modified the buffer. If FALSE, return a reference to the original buffer instead (to save space, not time) */ @@ -6680,123 +8125,207 @@ Py_DECREF(u); return (PyObject*) self; } - return (PyObject*) u; -} - -static int + else if (maxchar_new == maxchar_old) { + return u; + } + else { + /* In case the maximum character changed, we need to + convert the string to the new category. */ + PyObject *v = PyUnicode_New( + PyUnicode_GET_LENGTH(self), maxchar_new); + if (v == NULL) { + Py_DECREF(u); + return NULL; + } + if (maxchar_new > maxchar_old) { + /* If the maxchar increased so that the kind changed, not all + characters are representable anymore and we need to fix the + string again. This only happens in very few cases. */ + PyUnicode_CopyCharacters(v, 0, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)); + maxchar_old = fixfct((PyUnicodeObject*)v); + assert(maxchar_old > 0 && maxchar_old <= maxchar_new); + } + else + PyUnicode_CopyCharacters(v, 0, u, 0, PyUnicode_GET_LENGTH(self)); + + Py_DECREF(u); + return v; + } +} + +static Py_UCS4 fixupper(PyUnicodeObject *self) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; - - while (len-- > 0) { - register Py_UNICODE ch; - - ch = Py_UNICODE_TOUPPER(*s); - if (ch != *s) { - status = 1; - *s = ch; - } - s++; - } - - return status; -} - -static int + /* No need to call PyUnicode_READY(self) because this function is only + called as a callback from fixup() which does it already. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + int touched = 0; + Py_UCS4 maxchar = 0; + Py_ssize_t i; + + for (i = 0; i < len; ++i) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + const Py_UCS4 up = Py_UNICODE_TOUPPER(ch); + if (up != ch) { + if (up > maxchar) + maxchar = up; + PyUnicode_WRITE(kind, data, i, up); + touched = 1; + } + else if (ch > maxchar) + maxchar = ch; + } + + if (touched) + return maxchar; + else + return 0; +} + +static Py_UCS4 fixlower(PyUnicodeObject *self) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; - - while (len-- > 0) { - register Py_UNICODE ch; - - ch = Py_UNICODE_TOLOWER(*s); - if (ch != *s) { - status = 1; - *s = ch; - } - s++; - } - - return status; -} - -static int + /* No need to call PyUnicode_READY(self) because fixup() which does it. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + int touched = 0; + Py_UCS4 maxchar = 0; + Py_ssize_t i; + + for(i = 0; i < len; ++i) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + const Py_UCS4 lo = Py_UNICODE_TOLOWER(ch); + if (lo != ch) { + if (lo > maxchar) + maxchar = lo; + PyUnicode_WRITE(kind, data, i, lo); + touched = 1; + } + else if (ch > maxchar) + maxchar = ch; + } + + if (touched) + return maxchar; + else + return 0; +} + +static Py_UCS4 fixswapcase(PyUnicodeObject *self) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; - - while (len-- > 0) { - if (Py_UNICODE_ISUPPER(*s)) { - *s = Py_UNICODE_TOLOWER(*s); - status = 1; - } else if (Py_UNICODE_ISLOWER(*s)) { - *s = Py_UNICODE_TOUPPER(*s); - status = 1; - } - s++; - } - - return status; -} - -static int + /* No need to call PyUnicode_READY(self) because fixup() which does it. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + int touched = 0; + Py_UCS4 maxchar = 0; + Py_ssize_t i; + + for(i = 0; i < len; ++i) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + Py_UCS4 nu = 0; + + if (Py_UNICODE_ISUPPER(ch)) + nu = Py_UNICODE_TOLOWER(ch); + else if (Py_UNICODE_ISLOWER(ch)) + nu = Py_UNICODE_TOUPPER(ch); + + if (nu != 0) { + if (nu > maxchar) + maxchar = nu; + PyUnicode_WRITE(kind, data, i, nu); + touched = 1; + } + else if (ch > maxchar) + maxchar = ch; + } + + if (touched) + return maxchar; + else + return 0; +} + +static Py_UCS4 fixcapitalize(PyUnicodeObject *self) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; + /* No need to call PyUnicode_READY(self) because fixup() which does it. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + int touched = 0; + Py_UCS4 maxchar = 0; + Py_ssize_t i = 0; + Py_UCS4 ch; if (len == 0) return 0; - if (!Py_UNICODE_ISUPPER(*s)) { - *s = Py_UNICODE_TOUPPER(*s); - status = 1; - } - s++; - while (--len > 0) { - if (!Py_UNICODE_ISLOWER(*s)) { - *s = Py_UNICODE_TOLOWER(*s); - status = 1; - } - s++; - } - return status; -} - -static int + + ch = PyUnicode_READ(kind, data, i); + if (!Py_UNICODE_ISUPPER(ch)) { + maxchar = Py_UNICODE_TOUPPER(ch); + PyUnicode_WRITE(kind, data, i, maxchar); + touched = 1; + } + ++i; + for(; i < len; ++i) { + ch = PyUnicode_READ(kind, data, i); + if (!Py_UNICODE_ISLOWER(ch)) { + const Py_UCS4 lo = Py_UNICODE_TOLOWER(ch); + if (lo > maxchar) + maxchar = lo; + PyUnicode_WRITE(kind, data, i, lo); + touched = 1; + } + else if (ch > maxchar) + maxchar = ch; + } + + if (touched) + return maxchar; + else + return 0; +} + +static Py_UCS4 fixtitle(PyUnicodeObject *self) { - register Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register Py_UNICODE *e; + /* No need to call PyUnicode_READY(self) because fixup() which does it. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + Py_UCS4 maxchar = 0; + Py_ssize_t i = 0; int previous_is_cased; /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) { - Py_UNICODE ch = Py_UNICODE_TOTITLE(*p); - if (*p != ch) { - *p = ch; - return 1; + if (len == 1) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + const Py_UCS4 ti = Py_UNICODE_TOTITLE(ch); + if (ti != ch) { + PyUnicode_WRITE(kind, data, i, ti); + return ti; } else return 0; } - - e = p + PyUnicode_GET_SIZE(self); previous_is_cased = 0; - for (; p < e; p++) { - register const Py_UNICODE ch = *p; + for(; i < len; ++i) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + Py_UCS4 nu; if (previous_is_cased) - *p = Py_UNICODE_TOLOWER(ch); + nu = Py_UNICODE_TOLOWER(ch); else - *p = Py_UNICODE_TOTITLE(ch); + nu = Py_UNICODE_TOTITLE(ch); + + if (nu > maxchar) + maxchar = nu; + PyUnicode_WRITE(kind, data, i, nu); if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISUPPER(ch) || @@ -6805,22 +8334,22 @@ else previous_is_cased = 0; } - return 1; + return maxchar; } PyObject * PyUnicode_Join(PyObject *separator, PyObject *seq) { - const Py_UNICODE blank = ' '; - const Py_UNICODE *sep = ␣ + PyObject *sep = NULL; Py_ssize_t seplen = 1; - PyUnicodeObject *res = NULL; /* the result */ - Py_UNICODE *res_p; /* pointer to free byte in res's string area */ + PyObject *res = NULL; /* the result */ PyObject *fseq; /* PySequence_Fast(seq) */ Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */ PyObject **items; PyObject *item; - Py_ssize_t sz, i; + Py_ssize_t sz, i, res_offset; + Py_UCS4 maxchar = 0; + Py_UCS4 item_maxchar; fseq = PySequence_Fast(seq, ""); if (fseq == NULL) { @@ -6834,7 +8363,7 @@ seqlen = PySequence_Fast_GET_SIZE(fseq); /* If empty sequence, return u"". */ if (seqlen == 0) { - res = _PyUnicode_New(0); /* empty sequence; return u"" */ + res = PyUnicode_New(0, 0); goto Done; } items = PySequence_Fast_ITEMS(fseq); @@ -6843,15 +8372,17 @@ item = items[0]; if (PyUnicode_CheckExact(item)) { Py_INCREF(item); - res = (PyUnicodeObject *)item; + res = item; goto Done; } } else { /* Set up sep and seplen */ if (separator == NULL) { - sep = ␣ - seplen = 1; + /* fall back to a blank space separator */ + sep = PyUnicode_FromOrdinal(' '); + if (!sep || PyUnicode_READY(sep) == -1) + goto onError; } else { if (!PyUnicode_Check(separator)) { @@ -6861,8 +8392,14 @@ Py_TYPE(separator)->tp_name); goto onError; } - sep = PyUnicode_AS_UNICODE(separator); - seplen = PyUnicode_GET_SIZE(separator); + if (PyUnicode_READY(separator) == -1) + goto onError; + sep = separator; + seplen = PyUnicode_GET_LENGTH(separator); + maxchar = PyUnicode_MAX_CHAR_VALUE(separator); + /* inc refcount to keep this code path symetric with the + above case of a blank separator */ + Py_INCREF(sep); } } @@ -6882,7 +8419,12 @@ i, Py_TYPE(item)->tp_name); goto onError; } - sz += PyUnicode_GET_SIZE(item); + if (PyUnicode_READY(item) == -1) + goto onError; + sz += PyUnicode_GET_LENGTH(item); + item_maxchar = PyUnicode_MAX_CHAR_VALUE(item); + if (item_maxchar > maxchar) + maxchar = item_maxchar; if (i != 0) sz += seplen; if (sz < old_sz || sz > PY_SSIZE_T_MAX) { @@ -6892,42 +8434,70 @@ } } - res = _PyUnicode_New(sz); + res = PyUnicode_New(sz, maxchar); if (res == NULL) goto onError; /* Catenate everything. */ - res_p = PyUnicode_AS_UNICODE(res); - for (i = 0; i < seqlen; ++i) { + for (i = 0, res_offset = 0; i < seqlen; ++i) { Py_ssize_t itemlen; item = items[i]; - itemlen = PyUnicode_GET_SIZE(item); + itemlen = PyUnicode_GET_LENGTH(item); /* Copy item, and maybe the separator. */ if (i) { - Py_UNICODE_COPY(res_p, sep, seplen); - res_p += seplen; - } - Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen); - res_p += itemlen; - } + PyUnicode_CopyCharacters(res, res_offset, + sep, 0, seplen); + res_offset += seplen; + } + PyUnicode_CopyCharacters(res, res_offset, + item, 0, itemlen); + res_offset += itemlen; + } + assert(res_offset == PyUnicode_GET_LENGTH(res)); Done: Py_DECREF(fseq); - return (PyObject *)res; + Py_XDECREF(sep); + return res; onError: Py_DECREF(fseq); + Py_XDECREF(sep); Py_XDECREF(res); return NULL; } +#define FILL(kind, data, value, start, length) \ + do { \ + Py_ssize_t i_ = 0; \ + assert(kind != PyUnicode_WCHAR_KIND); \ + switch ((kind)) { \ + case PyUnicode_1BYTE_KIND: { \ + unsigned char * to_ = (unsigned char *)((data)) + (start); \ + memset(to_, (unsigned char)value, length); \ + break; \ + } \ + case PyUnicode_2BYTE_KIND: { \ + Py_UCS2 * to_ = (Py_UCS2 *)((data)) + (start); \ + for (; i_ < (length); ++i_, ++to_) *to_ = (value); \ + break; \ + } \ + default: { \ + Py_UCS4 * to_ = (Py_UCS4 *)((data)) + (start); \ + for (; i_ < (length); ++i_, ++to_) *to_ = (value); \ + break; \ + } \ + } \ + } while (0) + static PyUnicodeObject * pad(PyUnicodeObject *self, Py_ssize_t left, Py_ssize_t right, - Py_UNICODE fill) -{ - PyUnicodeObject *u; + Py_UCS4 fill) +{ + PyObject *u; + Py_UCS4 maxchar; if (left < 0) left = 0; @@ -6939,22 +8509,28 @@ return self; } - if (left > PY_SSIZE_T_MAX - self->length || - right > PY_SSIZE_T_MAX - (left + self->length)) { + if (left > PY_SSIZE_T_MAX - _PyUnicode_LENGTH(self) || + right > PY_SSIZE_T_MAX - (left + _PyUnicode_LENGTH(self))) { PyErr_SetString(PyExc_OverflowError, "padded string is too long"); return NULL; } - u = _PyUnicode_New(left + self->length + right); + maxchar = PyUnicode_MAX_CHAR_VALUE(self); + if (fill > maxchar) + maxchar = fill; + u = PyUnicode_New(left + _PyUnicode_LENGTH(self) + right, maxchar); if (u) { + int kind = PyUnicode_KIND(u); + void *data = PyUnicode_DATA(u); if (left) - Py_UNICODE_FILL(u->str, fill, left); - Py_UNICODE_COPY(u->str + left, self->str, self->length); + FILL(kind, data, fill, 0, left); if (right) - Py_UNICODE_FILL(u->str + left + self->length, fill, right); - } - - return u; -} + FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + PyUnicode_CopyCharacters(u, left, (PyObject*)self, 0, _PyUnicode_LENGTH(self)); + } + + return (PyUnicodeObject*)u; +} +#undef FILL PyObject * PyUnicode_Splitlines(PyObject *string, int keepends) @@ -6962,13 +8538,29 @@ PyObject *list; string = PyUnicode_FromObject(string); - if (string == NULL) - return NULL; - - list = stringlib_splitlines( - (PyObject*) string, PyUnicode_AS_UNICODE(string), - PyUnicode_GET_SIZE(string), keepends); - + if (string == NULL || PyUnicode_READY(string) == -1) + return NULL; + + switch(PyUnicode_KIND(string)) { + case PyUnicode_1BYTE_KIND: + list = ucs1lib_splitlines( + (PyObject*) string, PyUnicode_1BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + case PyUnicode_2BYTE_KIND: + list = ucs2lib_splitlines( + (PyObject*) string, PyUnicode_2BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + case PyUnicode_4BYTE_KIND: + list = ucs4lib_splitlines( + (PyObject*) string, PyUnicode_4BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + default: + assert(0); + list = 0; + } Py_DECREF(string); return list; } @@ -6978,19 +8570,81 @@ PyUnicodeObject *substring, Py_ssize_t maxcount) { + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; + PyObject* out; + if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; + if (PyUnicode_READY(self) == -1) + return NULL; + if (substring == NULL) - return stringlib_split_whitespace( - (PyObject*) self, self->str, self->length, maxcount - ); - - return stringlib_split( - (PyObject*) self, self->str, self->length, - substring->str, substring->length, - maxcount - ); + switch(PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: + return ucs1lib_split_whitespace( + (PyObject*) self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_2BYTE_KIND: + return ucs2lib_split_whitespace( + (PyObject*) self, PyUnicode_2BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_4BYTE_KIND: + return ucs4lib_split_whitespace( + (PyObject*) self, PyUnicode_4BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + default: + assert(0); + return NULL; + } + + if (PyUnicode_READY(substring) == -1) + return NULL; + + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind((PyObject*)self, kind); + if (!buf1) + return NULL; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind((PyObject*)substring, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + switch(kind) { + case PyUnicode_1BYTE_KIND: + out = ucs1lib_split( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_split( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_split( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + default: + out = NULL; + } + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return out; } static PyObject * @@ -6998,159 +8652,366 @@ PyUnicodeObject *substring, Py_ssize_t maxcount) { + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; + PyObject* out; + if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; + if (PyUnicode_READY(self) == -1) + return NULL; + if (substring == NULL) - return stringlib_rsplit_whitespace( - (PyObject*) self, self->str, self->length, maxcount - ); - - return stringlib_rsplit( - (PyObject*) self, self->str, self->length, - substring->str, substring->length, - maxcount - ); + switch(PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: + return ucs1lib_rsplit_whitespace( + (PyObject*) self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_2BYTE_KIND: + return ucs2lib_rsplit_whitespace( + (PyObject*) self, PyUnicode_2BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_4BYTE_KIND: + return ucs4lib_rsplit_whitespace( + (PyObject*) self, PyUnicode_4BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + default: + assert(0); + return NULL; + } + + if (PyUnicode_READY(substring) == -1) + return NULL; + + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind((PyObject*)self, kind); + if (!buf1) + return NULL; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind((PyObject*)substring, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + switch(kind) { + case PyUnicode_1BYTE_KIND: + out = ucs1lib_rsplit( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_rsplit( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_rsplit( + (PyObject*) self, buf1, len1, buf2, len2, maxcount); + break; + default: + out = NULL; + } + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return out; +} + +static Py_ssize_t +anylib_find(int kind, void *buf1, Py_ssize_t len1, + void *buf2, Py_ssize_t len2, Py_ssize_t offset) +{ + switch(kind) { + case PyUnicode_1BYTE_KIND: + return ucs1lib_find(buf1, len1, buf2, len2, offset); + case PyUnicode_2BYTE_KIND: + return ucs2lib_find(buf1, len1, buf2, len2, offset); + case PyUnicode_4BYTE_KIND: + return ucs4lib_find(buf1, len1, buf2, len2, offset); + } + assert(0); + return -1; +} + +static Py_ssize_t +anylib_count(int kind, void* sbuf, Py_ssize_t slen, + void *buf1, Py_ssize_t len1, Py_ssize_t maxcount) +{ + switch(kind) { + case PyUnicode_1BYTE_KIND: + return ucs1lib_count(sbuf, slen, buf1, len1, maxcount); + case PyUnicode_2BYTE_KIND: + return ucs2lib_count(sbuf, slen, buf1, len1, maxcount); + case PyUnicode_4BYTE_KIND: + return ucs4lib_count(sbuf, slen, buf1, len1, maxcount); + } + assert(0); + return 0; } static PyObject * -replace(PyUnicodeObject *self, - PyUnicodeObject *str1, - PyUnicodeObject *str2, - Py_ssize_t maxcount) -{ - PyUnicodeObject *u; +replace(PyObject *self, PyObject *str1, + PyObject *str2, Py_ssize_t maxcount) +{ + PyObject *u; + char *sbuf = PyUnicode_DATA(self); + char *buf1 = PyUnicode_DATA(str1); + char *buf2 = PyUnicode_DATA(str2); + int srelease = 0, release1 = 0, release2 = 0; + int skind = PyUnicode_KIND(self); + int kind1 = PyUnicode_KIND(str1); + int kind2 = PyUnicode_KIND(str2); + Py_ssize_t slen = PyUnicode_GET_LENGTH(self); + Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1); + Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2); if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; - else if (maxcount == 0 || self->length == 0) + else if (maxcount == 0 || slen == 0) goto nothing; - if (str1->length == str2->length) { + if (skind < kind1) + /* substring too wide to be present */ + goto nothing; + + if (len1 == len2) { Py_ssize_t i; /* same length */ - if (str1->length == 0) + if (len1 == 0) goto nothing; - if (str1->length == 1) { + if (len1 == 1) { /* replace characters */ - Py_UNICODE u1, u2; - if (!findchar(self->str, self->length, str1->str[0])) + Py_UCS4 u1, u2, maxchar; + int mayshrink, rkind; + u1 = PyUnicode_READ_CHAR(str1, 0); + if (!findchar(sbuf, PyUnicode_KIND(self), + slen, u1, 1)) goto nothing; - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + u2 = PyUnicode_READ_CHAR(str2, 0); + maxchar = PyUnicode_MAX_CHAR_VALUE(self); + /* Replacing u1 with u2 may cause a maxchar reduction in the + result string. */ + mayshrink = maxchar > 127; + if (u2 > maxchar) { + maxchar = u2; + mayshrink = 0; + } + u = PyUnicode_New(slen, maxchar); if (!u) - return NULL; - Py_UNICODE_COPY(u->str, self->str, self->length); - u1 = str1->str[0]; - u2 = str2->str[0]; - for (i = 0; i < u->length; i++) - if (u->str[i] == u1) { + goto error; + PyUnicode_CopyCharacters(u, 0, + (PyObject*)self, 0, slen); + rkind = PyUnicode_KIND(u); + for (i = 0; i < PyUnicode_GET_LENGTH(u); i++) + if (PyUnicode_READ(rkind, PyUnicode_DATA(u), i) == u1) { if (--maxcount < 0) break; - u->str[i] = u2; + PyUnicode_WRITE(rkind, PyUnicode_DATA(u), i, u2); } + if (mayshrink) { + PyObject *tmp = u; + u = PyUnicode_FromKindAndData(rkind, PyUnicode_DATA(tmp), + PyUnicode_GET_LENGTH(tmp)); + Py_DECREF(tmp); + } } else { - i = stringlib_find( - self->str, self->length, str1->str, str1->length, 0 - ); + int rkind = skind; + char *res; + if (kind1 < rkind) { + /* widen substring */ + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + i = anylib_find(rkind, sbuf, slen, buf1, len1, 0); if (i < 0) goto nothing; - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); - if (!u) - return NULL; - Py_UNICODE_COPY(u->str, self->str, self->length); - + if (rkind > kind2) { + /* widen replacement */ + buf2 = _PyUnicode_AsKind(str2, rkind); + if (!buf2) goto error; + release2 = 1; + } + else if (rkind < kind2) { + /* widen self and buf1 */ + rkind = kind2; + if (release1) PyMem_Free(buf1); + sbuf = _PyUnicode_AsKind(self, rkind); + if (!sbuf) goto error; + srelease = 1; + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + res = PyMem_Malloc(PyUnicode_KIND_SIZE(rkind, slen)); + if (!res) { + PyErr_NoMemory(); + goto error; + } + memcpy(res, sbuf, PyUnicode_KIND_SIZE(rkind, slen)); /* change everything in-place, starting with this one */ - Py_UNICODE_COPY(u->str+i, str2->str, str2->length); - i += str1->length; + memcpy(res + PyUnicode_KIND_SIZE(rkind, i), + buf2, + PyUnicode_KIND_SIZE(rkind, len2)); + i += len1; while ( --maxcount > 0) { - i = stringlib_find(self->str+i, self->length-i, - str1->str, str1->length, - i); + i = anylib_find(rkind, sbuf+PyUnicode_KIND_SIZE(rkind, i), + slen-i, + buf1, len1, i); if (i == -1) break; - Py_UNICODE_COPY(u->str+i, str2->str, str2->length); - i += str1->length; - } + memcpy(res + PyUnicode_KIND_SIZE(rkind, i), + buf2, + PyUnicode_KIND_SIZE(rkind, len2)); + i += len1; + } + + u = PyUnicode_FromKindAndData(rkind, res, slen); + PyMem_Free(res); + if (!u) goto error; } } else { - Py_ssize_t n, i, j; - Py_ssize_t product, new_size, delta; - Py_UNICODE *p; - - /* replace strings */ - n = stringlib_count(self->str, self->length, str1->str, str1->length, - maxcount); + Py_ssize_t n, i, j, ires; + Py_ssize_t product, new_size; + int rkind = skind; + char *res; + + if (kind1 < rkind) { + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + n = anylib_count(rkind, sbuf, slen, buf1, len1, maxcount); if (n == 0) goto nothing; - /* new_size = self->length + n * (str2->length - str1->length)); */ - delta = (str2->length - str1->length); - if (delta == 0) { - new_size = self->length; - } else { - product = n * (str2->length - str1->length); - if ((product / (str2->length - str1->length)) != n) { + if (kind2 < rkind) { + buf2 = _PyUnicode_AsKind(str2, rkind); + if (!buf2) goto error; + release2 = 1; + } + else if (kind2 > rkind) { + rkind = kind2; + sbuf = _PyUnicode_AsKind(self, rkind); + if (!sbuf) goto error; + srelease = 1; + if (release1) PyMem_Free(buf1); + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + /* new_size = PyUnicode_GET_LENGTH(self) + n * (PyUnicode_GET_LENGTH(str2) - + PyUnicode_GET_LENGTH(str1))); */ + product = n * (len2-len1); + if ((product / (len2-len1)) != n) { PyErr_SetString(PyExc_OverflowError, "replace string is too long"); - return NULL; - } - new_size = self->length + product; - if (new_size < 0) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - } - u = _PyUnicode_New(new_size); - if (!u) - return NULL; - i = 0; - p = u->str; - if (str1->length > 0) { + goto error; + } + new_size = slen + product; + if (new_size < 0 || new_size > (PY_SSIZE_T_MAX >> (rkind-1))) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + goto error; + } + res = PyMem_Malloc(PyUnicode_KIND_SIZE(rkind, new_size)); + if (!res) + goto error; + ires = i = 0; + if (len1 > 0) { while (n-- > 0) { /* look for next match */ - j = stringlib_find(self->str+i, self->length-i, - str1->str, str1->length, - i); + j = anylib_find(rkind, + sbuf + PyUnicode_KIND_SIZE(rkind, i), + slen-i, buf1, len1, i); if (j == -1) break; else if (j > i) { /* copy unchanged part [i:j] */ - Py_UNICODE_COPY(p, self->str+i, j-i); - p += j - i; + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + sbuf + PyUnicode_KIND_SIZE(rkind, i), + PyUnicode_KIND_SIZE(rkind, j-i)); + ires += j - i; } /* copy substitution string */ - if (str2->length > 0) { - Py_UNICODE_COPY(p, str2->str, str2->length); - p += str2->length; + if (len2 > 0) { + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + buf2, + PyUnicode_KIND_SIZE(rkind, len2)); + ires += len2; } - i = j + str1->length; - } - if (i < self->length) + i = j + len1; + } + if (i < slen) /* copy tail [i:] */ - Py_UNICODE_COPY(p, self->str+i, self->length-i); + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + sbuf + PyUnicode_KIND_SIZE(rkind, i), + PyUnicode_KIND_SIZE(rkind, slen-i)); } else { /* interleave */ while (n > 0) { - Py_UNICODE_COPY(p, str2->str, str2->length); - p += str2->length; + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + buf2, + PyUnicode_KIND_SIZE(rkind, len2)); + ires += len2; if (--n <= 0) break; - *p++ = self->str[i++]; - } - Py_UNICODE_COPY(p, self->str+i, self->length-i); - } - } - return (PyObject *) u; + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + sbuf + PyUnicode_KIND_SIZE(rkind, i), + PyUnicode_KIND_SIZE(rkind, 1)); + ires++; + i++; + } + memcpy(res + PyUnicode_KIND_SIZE(rkind, ires), + sbuf + PyUnicode_KIND_SIZE(rkind, i), + PyUnicode_KIND_SIZE(rkind, slen-i)); + } + u = PyUnicode_FromKindAndData(rkind, res, new_size); + } + if (srelease) + PyMem_FREE(sbuf); + if (release1) + PyMem_FREE(buf1); + if (release2) + PyMem_FREE(buf2); + return u; nothing: /* nothing to replace; return original string (when possible) */ + if (srelease) + PyMem_FREE(sbuf); + if (release1) + PyMem_FREE(buf1); + if (release2) + PyMem_FREE(buf2); if (PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject *) self; } - return PyUnicode_FromUnicode(self->str, self->length); + return PyUnicode_FromKindAndData(PyUnicode_KIND(self), + PyUnicode_DATA(self), + PyUnicode_GET_LENGTH(self)); + error: + if (srelease && sbuf) + PyMem_FREE(sbuf); + if (release1 && buf1) + PyMem_FREE(buf1); + if (release2 && buf2) + PyMem_FREE(buf2); + return NULL; } /* --- Unicode Object Methods --------------------------------------------- */ @@ -7222,9 +9083,8 @@ static int convert_uc(PyObject *obj, void *addr) { - Py_UNICODE *fillcharloc = (Py_UNICODE *)addr; + Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; PyObject *uniobj; - Py_UNICODE *unistr; uniobj = PyUnicode_FromObject(obj); if (uniobj == NULL) { @@ -7232,14 +9092,17 @@ "The fill character cannot be converted to Unicode"); return 0; } - if (PyUnicode_GET_SIZE(uniobj) != 1) { + if (PyUnicode_GET_LENGTH(uniobj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); Py_DECREF(uniobj); return 0; } - unistr = PyUnicode_AS_UNICODE(uniobj); - *fillcharloc = unistr[0]; + if (PyUnicode_READY(uniobj)) { + Py_DECREF(uniobj); + return 0; + } + *fillcharloc = PyUnicode_READ_CHAR(uniobj, 0); Py_DECREF(uniobj); return 1; } @@ -7255,17 +9118,20 @@ { Py_ssize_t marg, left; Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; + + if (PyUnicode_READY(self) == -1) + return NULL; if (!PyArg_ParseTuple(args, "n|O&:center", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { + if (_PyUnicode_LENGTH(self) >= width && PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject*) self; } - marg = width - self->length; + marg = width - _PyUnicode_LENGTH(self); left = marg / 2 + (marg & width & 1); return (PyObject*) pad(self, left, marg - left, fillchar); @@ -7297,8 +9163,8 @@ Py_UNICODE *s1 = str1->str; Py_UNICODE *s2 = str2->str; - len1 = str1->length; - len2 = str2->length; + len1 = str1->_base._base.length; + len2 = str2->_base._base.length; while (len1 > 0 && len2 > 0) { Py_UNICODE c1, c2; @@ -7323,27 +9189,29 @@ #else +/* This function assumes that str1 and str2 are readied by the caller. */ + static int unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2) { - register Py_ssize_t len1, len2; - - Py_UNICODE *s1 = str1->str; - Py_UNICODE *s2 = str2->str; - - len1 = str1->length; - len2 = str2->length; - - while (len1 > 0 && len2 > 0) { - Py_UNICODE c1, c2; - - c1 = *s1++; - c2 = *s2++; + int kind1, kind2; + void *data1, *data2; + Py_ssize_t len1, len2, i; + + kind1 = PyUnicode_KIND(str1); + kind2 = PyUnicode_KIND(str2); + data1 = PyUnicode_DATA(str1); + data2 = PyUnicode_DATA(str2); + len1 = PyUnicode_GET_LENGTH(str1); + len2 = PyUnicode_GET_LENGTH(str2); + + for (i = 0; i < len1 && i < len2; ++i) { + Py_UCS4 c1, c2; + c1 = PyUnicode_READ(kind1, data1, i); + c2 = PyUnicode_READ(kind2, data2, i); if (c1 != c2) return (c1 < c2) ? -1 : 1; - - len1--; len2--; } return (len1 < len2) ? -1 : (len1 != len2); @@ -7354,9 +9222,13 @@ int PyUnicode_Compare(PyObject *left, PyObject *right) { - if (PyUnicode_Check(left) && PyUnicode_Check(right)) + if (PyUnicode_Check(left) && PyUnicode_Check(right)) { + if (PyUnicode_READY(left) == -1 || + PyUnicode_READY(right) == -1) + return -1; return unicode_compare((PyUnicodeObject *)left, (PyUnicodeObject *)right); + } PyErr_Format(PyExc_TypeError, "Can't compare %.100s and %.100s", left->ob_type->tp_name, @@ -7367,17 +9239,23 @@ int PyUnicode_CompareWithASCIIString(PyObject* uni, const char* str) { - int i; - Py_UNICODE *id; + Py_ssize_t i; + int kind; + void *data; + Py_UCS4 chr; + assert(PyUnicode_Check(uni)); - id = PyUnicode_AS_UNICODE(uni); + if (PyUnicode_READY(uni) == -1) + return -1; + kind = PyUnicode_KIND(uni); + data = PyUnicode_DATA(uni); /* Compare Unicode string and source character set string */ - for (i = 0; id[i] && str[i]; i++) - if (id[i] != str[i]) - return ((int)id[i] < (int)str[i]) ? -1 : 1; + for (i = 0; (chr = PyUnicode_READ(kind, data, i)) && str[i]; i++) + if (chr != str[i]) + return (chr < (unsigned char)(str[i])) ? -1 : 1; /* This check keeps Python strings that end in '\0' from comparing equal to C strings identical up to that point. */ - if (PyUnicode_GET_SIZE(uni) != i || id[i]) + if (PyUnicode_GET_LENGTH(uni) != i || chr) return 1; /* uni is longer */ if (str[i]) return -1; /* str is longer */ @@ -7395,7 +9273,11 @@ if (PyUnicode_Check(left) && PyUnicode_Check(right)) { PyObject *v; - if (PyUnicode_GET_SIZE(left) != PyUnicode_GET_SIZE(right)) { + if (PyUnicode_READY(left) == -1 || + PyUnicode_READY(right) == -1) + return NULL; + if (PyUnicode_GET_LENGTH(left) != PyUnicode_GET_LENGTH(right) || + PyUnicode_KIND(left) != PyUnicode_KIND(right)) { if (op == Py_EQ) { Py_INCREF(Py_False); return Py_False; @@ -7446,6 +9328,9 @@ PyUnicode_Contains(PyObject *container, PyObject *element) { PyObject *str, *sub; + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; int result; /* Coerce the two arguments */ @@ -7456,18 +9341,59 @@ element->ob_type->tp_name); return -1; } + if (PyUnicode_READY(sub) == -1) + return -1; str = PyUnicode_FromObject(container); - if (!str) { + if (!str || PyUnicode_READY(container) == -1) { Py_DECREF(sub); return -1; } - result = stringlib_contains_obj(str, sub); + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(sub); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(sub); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind((PyObject*)str, kind); + if (!buf1) { + Py_DECREF(sub); + return -1; + } + if (kind2 != kind) + buf2 = _PyUnicode_AsKind((PyObject*)sub, kind); + if (!buf2) { + Py_DECREF(sub); + if (kind1 != kind) PyMem_Free(buf1); + return -1; + } + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(sub); + + switch(kind) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + default: + result = -1; + assert(0); + } Py_DECREF(str); Py_DECREF(sub); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return result; } @@ -7476,36 +9402,46 @@ PyObject * PyUnicode_Concat(PyObject *left, PyObject *right) { - PyUnicodeObject *u = NULL, *v = NULL, *w; + PyObject *u = NULL, *v = NULL, *w; + Py_UCS4 maxchar; /* Coerce the two arguments */ - u = (PyUnicodeObject *)PyUnicode_FromObject(left); + u = PyUnicode_FromObject(left); if (u == NULL) goto onError; - v = (PyUnicodeObject *)PyUnicode_FromObject(right); + v = PyUnicode_FromObject(right); if (v == NULL) goto onError; /* Shortcuts */ - if (v == unicode_empty) { + if (v == (PyObject*)unicode_empty) { Py_DECREF(v); - return (PyObject *)u; - } - if (u == unicode_empty) { + return u; + } + if (u == (PyObject*)unicode_empty) { Py_DECREF(u); - return (PyObject *)v; - } + return v; + } + + if (PyUnicode_READY(u) == -1 || PyUnicode_READY(v) == -1) + goto onError; + + maxchar = PyUnicode_MAX_CHAR_VALUE(u); + if (PyUnicode_MAX_CHAR_VALUE(v) > maxchar) + maxchar = PyUnicode_MAX_CHAR_VALUE(v); /* Concat the two Unicode strings */ - w = _PyUnicode_New(u->length + v->length); + w = PyUnicode_New( + PyUnicode_GET_LENGTH(u) + PyUnicode_GET_LENGTH(v), + maxchar); if (w == NULL) goto onError; - Py_UNICODE_COPY(w->str, u->str, u->length); - Py_UNICODE_COPY(w->str + u->length, v->str, v->length); - + PyUnicode_CopyCharacters(w, 0, u, 0, PyUnicode_GET_LENGTH(u)); + PyUnicode_CopyCharacters(w, PyUnicode_GET_LENGTH(u), v, 0, + PyUnicode_GET_LENGTH(v)); Py_DECREF(u); Py_DECREF(v); - return (PyObject *)w; + return w; onError: Py_XDECREF(u); @@ -7550,17 +9486,65 @@ Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; PyObject *result; + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2, iresult; if (!stringlib_parse_args_finds_unicode("count", args, &substring, &start, &end)) return NULL; - ADJUST_INDICES(start, end, self->length); - result = PyLong_FromSsize_t( - stringlib_count(self->str + start, end - start, - substring->str, substring->length, - PY_SSIZE_T_MAX) - ); + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind((PyObject*)self, kind); + if (!buf1) { + Py_DECREF(substring); + return NULL; + } + if (kind2 != kind) + buf2 = _PyUnicode_AsKind((PyObject*)substring, kind); + if (!buf2) { + Py_DECREF(substring); + if (kind1 != kind) PyMem_Free(buf1); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + ADJUST_INDICES(start, end, len1); + switch(kind) { + case PyUnicode_1BYTE_KIND: + iresult = ucs1lib_count( + ((Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + iresult = ucs2lib_count( + ((Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + iresult = ucs4lib_count( + ((Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + assert(0); iresult = 0; + } + + result = PyLong_FromSsize_t(iresult); + + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); Py_DECREF(substring); @@ -7603,18 +9587,21 @@ Py_UNICODE *p; Py_UNICODE *q; Py_UNICODE *qe; - Py_ssize_t i, j, incr; + Py_ssize_t i, j, incr, wstr_length; PyUnicodeObject *u; int tabsize = 8; if (!PyArg_ParseTuple(args, "|i:expandtabs", &tabsize)) return NULL; + if (PyUnicode_AsUnicodeAndSize((PyObject *)self, &wstr_length) == NULL) + return NULL; + /* First pass: determine size of output string */ i = 0; /* chars up to and including most recent \n or \r */ j = 0; /* chars since most recent \n or \r (use in tab calculations) */ - e = self->str + self->length; /* end of input */ - for (p = self->str; p < e; p++) + e = _PyUnicode_WSTR(self) + wstr_length; /* end of input */ + for (p = _PyUnicode_WSTR(self); p < e; p++) if (*p == '\t') { if (tabsize > 0) { incr = tabsize - (j % tabsize); /* cannot overflow */ @@ -7644,10 +9631,10 @@ return NULL; j = 0; /* same as in first pass */ - q = u->str; /* next output char */ - qe = u->str + u->length; /* end of output */ - - for (p = self->str; p < e; p++) + q = _PyUnicode_WSTR(u); /* next output char */ + qe = _PyUnicode_WSTR(u) + PyUnicode_GET_SIZE(u); /* end of output */ + + for (p = _PyUnicode_WSTR(self); p < e; p++) if (*p == '\t') { if (tabsize > 0) { i = tabsize - (j % tabsize); @@ -7668,6 +9655,10 @@ j = 0; } + if (PyUnicode_READY(u) == -1) { + Py_DECREF(u); + return NULL; + } return (PyObject*) u; overflow2: @@ -7687,7 +9678,7 @@ Return -1 on failure."); static PyObject * -unicode_find(PyUnicodeObject *self, PyObject *args) +unicode_find(PyObject *self, PyObject *args) { PyUnicodeObject *substring; Py_ssize_t start; @@ -7698,26 +9689,38 @@ &start, &end)) return NULL; - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice( + ucs1lib_find_slice, ucs2lib_find_slice, ucs4lib_find_slice, + self, (PyObject*)substring, start, end ); Py_DECREF(substring); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } static PyObject * unicode_getitem(PyUnicodeObject *self, Py_ssize_t index) { - if (index < 0 || index >= self->length) { + Py_UCS4 ch; + + if (PyUnicode_READY(self) == -1) + return NULL; + if (index < 0 || index >= _PyUnicode_LENGTH(self)) { PyErr_SetString(PyExc_IndexError, "string index out of range"); return NULL; } - return (PyObject*) PyUnicode_FromUnicode(&self->str[index], 1); + ch = PyUnicode_READ(PyUnicode_KIND(self), PyUnicode_DATA(self), index); + return PyUnicode_FromOrdinal(ch); } /* Believe it or not, this produces the same value for ASCII strings @@ -7726,22 +9729,48 @@ unicode_hash(PyUnicodeObject *self) { Py_ssize_t len; - Py_UNICODE *p; Py_uhash_t x; - if (self->hash != -1) - return self->hash; - len = Py_SIZE(self); - p = self->str; - x = (Py_uhash_t)*p << 7; - while (--len >= 0) - x = (1000003U*x) ^ (Py_uhash_t)*p++; - x ^= (Py_uhash_t)Py_SIZE(self); + if (_PyUnicode_HASH(self) != -1) + return _PyUnicode_HASH(self); + if (PyUnicode_READY(self) == -1) + return -1; + len = PyUnicode_GET_LENGTH(self); + + /* The hash function as a macro, gets expanded three times below. */ +#define HASH(P) \ + x = (Py_uhash_t)*P << 7; \ + while (--len >= 0) \ + x = (1000003*x) ^ (Py_uhash_t)*P++; + + switch (PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: { + const unsigned char *c = PyUnicode_1BYTE_DATA(self); + HASH(c); + break; + } + case PyUnicode_2BYTE_KIND: { + const Py_UCS2 *s = PyUnicode_2BYTE_DATA(self); + HASH(s); + break; + } + default: { + Py_UCS4 *l; + assert(PyUnicode_KIND(self) == PyUnicode_4BYTE_KIND && + "Impossible switch case in unicode_hash"); + l = PyUnicode_4BYTE_DATA(self); + HASH(l); + break; + } + } + x ^= (Py_uhash_t)PyUnicode_GET_LENGTH(self); + if (x == -1) x = -2; - self->hash = x; + _PyUnicode_HASH(self) = x; return x; } +#undef HASH PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ @@ -7749,7 +9778,7 @@ Like S.find() but raise ValueError when the substring is not found."); static PyObject * -unicode_index(PyUnicodeObject *self, PyObject *args) +unicode_index(PyObject *self, PyObject *args) { Py_ssize_t result; PyUnicodeObject *substring; @@ -7760,14 +9789,21 @@ &start, &end)) return NULL; - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice( + ucs1lib_find_slice, ucs2lib_find_slice, ucs4lib_find_slice, + self, (PyObject*)substring, start, end ); Py_DECREF(substring); + if (result == -2) + return NULL; + if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; @@ -7785,22 +9821,29 @@ static PyObject* unicode_islower(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong(Py_UNICODE_ISLOWER(*p)); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISLOWER(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) return PyBool_FromLong(0); @@ -7819,22 +9862,29 @@ static PyObject* unicode_isupper(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong(Py_UNICODE_ISUPPER(*p) != 0); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISUPPER(PyUnicode_READ(kind, data, 0)) != 0); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch)) return PyBool_FromLong(0); @@ -7855,24 +9905,32 @@ static PyObject* unicode_istitle(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased, previous_is_cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong((Py_UNICODE_ISTITLE(*p) != 0) || - (Py_UNICODE_ISUPPER(*p) != 0)); + if (length == 1) { + Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong((Py_UNICODE_ISTITLE(ch) != 0) || + (Py_UNICODE_ISUPPER(ch) != 0)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; previous_is_cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) { if (previous_is_cased) @@ -7901,21 +9959,27 @@ static PyObject* unicode_isspace(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISSPACE(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (!Py_UNICODE_ISSPACE(ch)) return PyBool_FromLong(0); } @@ -7931,21 +9995,27 @@ static PyObject* unicode_isalpha(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISALPHA(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISALPHA(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -7960,21 +10030,29 @@ static PyObject* unicode_isalnum(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + int kind; + void *data; + Py_ssize_t len, i; + + if (PyUnicode_READY(self) == -1) + return NULL; + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISALNUM(*p)) - return PyBool_FromLong(1); + if (len == 1) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong(Py_UNICODE_ISALNUM(ch)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (len == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < len; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (!Py_UNICODE_ISALNUM(ch)) return PyBool_FromLong(0); } @@ -7990,21 +10068,27 @@ static PyObject* unicode_isdecimal(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISDECIMAL(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISDECIMAL(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -8019,21 +10103,28 @@ static PyObject* unicode_isdigit(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISDIGIT(*p)) - return PyBool_FromLong(1); + if (length == 1) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong(Py_UNICODE_ISDIGIT(ch)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISDIGIT(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISDIGIT(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -8048,21 +10139,27 @@ static PyObject* unicode_isnumeric(PyUnicodeObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISNUMERIC(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISNUMERIC(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -8071,13 +10168,21 @@ int PyUnicode_IsIdentifier(PyObject *self) { - const Py_UNICODE *p = PyUnicode_AS_UNICODE((PyUnicodeObject*)self); - const Py_UNICODE *e; + int kind; + void *data; + Py_ssize_t i; Py_UCS4 first; + if (PyUnicode_READY(self) == -1) { + Py_FatalError("identifier not ready"); + return 0; + } + /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (PyUnicode_GET_LENGTH(self) == 0) return 0; + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* PEP 3131 says that the first character must be in XID_Start and subsequent characters in XID_Continue, @@ -8087,13 +10192,12 @@ definition of XID_Start and XID_Continue, it is sufficient to check just for these, except that _ must be allowed as starting an identifier. */ - e = p + PyUnicode_GET_SIZE(self); - first = _Py_UNICODE_NEXT(p, e); + first = PyUnicode_READ(kind, data, 0); if (!_PyUnicode_IsXidStart(first) && first != 0x5F /* LOW LINE */) return 0; - while (p < e) - if (!_PyUnicode_IsXidContinue(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < PyUnicode_GET_LENGTH(self); i++) + if (!_PyUnicode_IsXidContinue(PyUnicode_READ(kind, data, i))) return 0; return 1; } @@ -8119,17 +10223,23 @@ static PyObject* unicode_isprintable(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && Py_UNICODE_ISPRINTABLE(*p)) { - Py_RETURN_TRUE; - } - - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISPRINTABLE(_Py_UNICODE_NEXT(p, e))) { + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, 0))); + + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, i))) { Py_RETURN_FALSE; } } @@ -8151,7 +10261,9 @@ static Py_ssize_t unicode_length(PyUnicodeObject *self) { - return self->length; + if (PyUnicode_READY(self) == -1) + return -1; + return PyUnicode_GET_LENGTH(self); } PyDoc_STRVAR(ljust__doc__, @@ -8164,17 +10276,20 @@ unicode_ljust(PyUnicodeObject *self, PyObject *args) { Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; + + if (PyUnicode_READY(self) == -1) + return NULL; if (!PyArg_ParseTuple(args, "n|O&:ljust", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { + if (_PyUnicode_LENGTH(self) >= width && PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject*) self; } - return (PyObject*) pad(self, 0, width - self->length, fillchar); + return (PyObject*) pad(self, 0, width - _PyUnicode_LENGTH(self), fillchar); } PyDoc_STRVAR(lower__doc__, @@ -8201,17 +10316,25 @@ PyObject * _PyUnicode_XStrip(PyUnicodeObject *self, int striptype, PyObject *sepobj) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(self); - Py_ssize_t len = PyUnicode_GET_SIZE(self); - Py_UNICODE *sep = PyUnicode_AS_UNICODE(sepobj); - Py_ssize_t seplen = PyUnicode_GET_SIZE(sepobj); - Py_ssize_t i, j; - - BLOOM_MASK sepmask = make_bloom_mask(sep, seplen); + void *data; + int kind; + Py_ssize_t i, j, len; + BLOOM_MASK sepmask; + + if (PyUnicode_READY(self) == -1 || PyUnicode_READY(sepobj) == -1) + return NULL; + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); + sepmask = make_bloom_mask(PyUnicode_KIND(sepobj), + PyUnicode_DATA(sepobj), + PyUnicode_GET_LENGTH(sepobj)); i = 0; if (striptype != RIGHTSTRIP) { - while (i < len && BLOOM_MEMBER(sepmask, s[i], sep, seplen)) { + while (i < len && + BLOOM_MEMBER(sepmask, PyUnicode_READ(kind, data, i), sepobj)) { i++; } } @@ -8220,7 +10343,8 @@ if (striptype != LEFTSTRIP) { do { j--; - } while (j >= i && BLOOM_MEMBER(sepmask, s[j], sep, seplen)); + } while (j >= i && + BLOOM_MEMBER(sepmask, PyUnicode_READ(kind, data, j), sepobj)); j++; } @@ -8229,19 +10353,85 @@ return (PyObject*)self; } else - return PyUnicode_FromUnicode(s+i, j-i); -} - + return PyUnicode_Substring((PyObject*)self, i, j); +} + +/* Assumes an already ready self string. */ + +static PyObject * +substring(PyUnicodeObject *self, Py_ssize_t start, Py_ssize_t len) +{ + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + Py_UCS4 maxchar = 0; + Py_ssize_t i; + PyObject *unicode; + + if (start < 0 || len < 0 || (start + len) > PyUnicode_GET_LENGTH(self)) { + PyErr_BadInternalCall(); + return NULL; + } + + if (len == PyUnicode_GET_LENGTH(self) && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*)self; + } + + for (i = 0; i < len; ++i) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, start + i); + if (ch > maxchar) + maxchar = ch; + } + + unicode = PyUnicode_New(len, maxchar); + if (unicode == NULL) + return NULL; + PyUnicode_CopyCharacters(unicode, 0, + (PyObject*)self, start, len); + return unicode; +} + +PyObject* +PyUnicode_Substring(PyObject *self, Py_ssize_t start, Py_ssize_t end) +{ + unsigned char *data; + int kind; + + if (start == 0 && end == PyUnicode_GET_LENGTH(self) + && PyUnicode_CheckExact(self)) + { + Py_INCREF(self); + return (PyObject *)self; + } + + if ((end - start) == 1) + return unicode_getitem((PyUnicodeObject*)self, start); + + if (PyUnicode_READY(self) == -1) + return NULL; + kind = PyUnicode_KIND(self); + data = PyUnicode_1BYTE_DATA(self); + return PyUnicode_FromKindAndData(kind, data + PyUnicode_KIND_SIZE(kind, start), + end-start); +} static PyObject * do_strip(PyUnicodeObject *self, int striptype) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(self); - Py_ssize_t len = PyUnicode_GET_SIZE(self), i, j; + int kind; + void *data; + Py_ssize_t len, i, j; + + if (PyUnicode_READY(self) == -1) + return NULL; + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); i = 0; if (striptype != RIGHTSTRIP) { - while (i < len && Py_UNICODE_ISSPACE(s[i])) { + while (i < len && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, i))) { i++; } } @@ -8250,7 +10440,7 @@ if (striptype != LEFTSTRIP) { do { j--; - } while (j >= i && Py_UNICODE_ISSPACE(s[j])); + } while (j >= i && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, j))); j++; } @@ -8259,7 +10449,7 @@ return (PyObject*)self; } else - return PyUnicode_FromUnicode(s+i, j-i); + return substring(self, i, j-i); } @@ -8339,9 +10529,8 @@ unicode_repeat(PyUnicodeObject *str, Py_ssize_t len) { PyUnicodeObject *u; - Py_UNICODE *p; - Py_ssize_t nchars; - size_t nbytes; + Py_ssize_t nchars, n; + size_t nbytes, char_size; if (len < 1) { Py_INCREF(unicode_empty); @@ -8354,35 +10543,46 @@ return (PyObject*) str; } + if (PyUnicode_READY(str) == -1) + return NULL; + /* ensure # of chars needed doesn't overflow int and # of bytes * needed doesn't overflow size_t */ - nchars = len * str->length; - if (nchars / len != str->length) { + nchars = len * PyUnicode_GET_LENGTH(str); + if (nchars / len != PyUnicode_GET_LENGTH(str)) { PyErr_SetString(PyExc_OverflowError, "repeated string is too long"); return NULL; } - nbytes = (nchars + 1) * sizeof(Py_UNICODE); - if (nbytes / sizeof(Py_UNICODE) != (size_t)(nchars + 1)) { + char_size = PyUnicode_CHARACTER_SIZE(str); + nbytes = (nchars + 1) * char_size; + if (nbytes / char_size != (size_t)(nchars + 1)) { PyErr_SetString(PyExc_OverflowError, "repeated string is too long"); return NULL; } - u = _PyUnicode_New(nchars); + u = (PyUnicodeObject *)PyUnicode_New(nchars, PyUnicode_MAX_CHAR_VALUE(str)); if (!u) return NULL; - p = u->str; - - if (str->length == 1) { - Py_UNICODE_FILL(p, str->str[0], len); - } else { - Py_ssize_t done = str->length; /* number of characters copied this far */ - Py_UNICODE_COPY(p, str->str, str->length); + if (PyUnicode_GET_LENGTH(str) == 1) { + const int kind = PyUnicode_KIND(str); + const Py_UCS4 fill_char = PyUnicode_READ(kind, PyUnicode_DATA(str), 0); + void *to = PyUnicode_DATA(u); + for (n = 0; n < len; ++n) + PyUnicode_WRITE(kind, to, n, fill_char); + } + else { + /* number of characters copied this far */ + Py_ssize_t done = PyUnicode_GET_LENGTH(str); + const Py_ssize_t char_size = PyUnicode_CHARACTER_SIZE(str); + char *to = (char *) PyUnicode_DATA(u); + Py_MEMCPY(to, PyUnicode_DATA(str), + PyUnicode_GET_LENGTH(str) * char_size); while (done < nchars) { - Py_ssize_t n = (done <= nchars-done) ? done : nchars-done; - Py_UNICODE_COPY(p+done, p, n); + n = (done <= nchars-done) ? done : nchars-done; + Py_MEMCPY(to + (done * char_size), to, n * char_size); done += n; } } @@ -8402,23 +10602,20 @@ PyObject *result; self = PyUnicode_FromObject(obj); - if (self == NULL) + if (self == NULL || PyUnicode_READY(obj) == -1) return NULL; str1 = PyUnicode_FromObject(subobj); - if (str1 == NULL) { + if (str1 == NULL || PyUnicode_READY(obj) == -1) { Py_DECREF(self); return NULL; } str2 = PyUnicode_FromObject(replobj); - if (str2 == NULL) { + if (str2 == NULL || PyUnicode_READY(obj)) { Py_DECREF(self); Py_DECREF(str1); return NULL; } - result = replace((PyUnicodeObject *)self, - (PyUnicodeObject *)str1, - (PyUnicodeObject *)str2, - maxcount); + result = replace(self, str1, str2, maxcount); Py_DECREF(self); Py_DECREF(str1); Py_DECREF(str2); @@ -8433,20 +10630,22 @@ given, only the first count occurrences are replaced."); static PyObject* -unicode_replace(PyUnicodeObject *self, PyObject *args) -{ - PyUnicodeObject *str1; - PyUnicodeObject *str2; +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str1; + PyObject *str2; Py_ssize_t maxcount = -1; PyObject *result; if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) return NULL; - str1 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str1); - if (str1 == NULL) - return NULL; - str2 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str2); - if (str2 == NULL) { + if (!PyUnicode_READY(self) == -1) + return NULL; + str1 = PyUnicode_FromObject(str1); + if (str1 == NULL || PyUnicode_READY(str1) == -1) + return NULL; + str2 = PyUnicode_FromObject(str2); + if (str2 == NULL || PyUnicode_READY(str1) == -1) { Py_DECREF(str1); return NULL; } @@ -8462,148 +10661,149 @@ unicode_repr(PyObject *unicode) { PyObject *repr; - Py_UNICODE *p; - Py_UNICODE *s = PyUnicode_AS_UNICODE(unicode); - Py_ssize_t size = PyUnicode_GET_SIZE(unicode); - - /* XXX(nnorwitz): rather than over-allocating, it would be - better to choose a different scheme. Perhaps scan the - first N-chars of the string and allocate based on that size. - */ - /* Initial allocation is based on the longest-possible unichr - escape. - - In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source - unichr, so in this case it's the longest unichr escape. In - narrow (UTF-16) builds this is five chars per source unichr - since there are two unichrs in the surrogate pair, so in narrow - (UTF-16) builds it's not the longest unichr escape. - - In wide or narrow builds '\uxxxx' is 6 chars per source unichr, - so in the narrow (UTF-16) build case it's the longest unichr - escape. - */ - - repr = PyUnicode_FromUnicode(NULL, - 2 /* quotes */ -#ifdef Py_UNICODE_WIDE - + 10*size -#else - + 6*size -#endif - + 1); + Py_ssize_t isize; + Py_ssize_t osize, squote, dquote, i, o; + Py_UCS4 max, quote; + int ikind, okind; + void *idata, *odata; + + if (PyUnicode_READY(unicode) == -1) + return NULL; + + isize = PyUnicode_GET_LENGTH(unicode); + idata = PyUnicode_DATA(unicode); + + /* Compute length of output, quote characters, and + maximum character */ + osize = 2; /* quotes */ + max = 127; + squote = dquote = 0; + ikind = PyUnicode_KIND(unicode); + for (i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + switch (ch) { + case '\'': squote++; osize++; break; + case '"': dquote++; osize++; break; + case '\\': case '\t': case '\r': case '\n': + osize += 2; break; + default: + /* Fast-path ASCII */ + if (ch < ' ' || ch == 0x7f) + osize += 4; /* \xHH */ + else if (ch < 0x7f) + osize++; + else if (Py_UNICODE_ISPRINTABLE(ch)) { + osize++; + max = ch > max ? ch : max; + } + else if (ch < 0x100) + osize += 4; /* \xHH */ + else if (ch < 0x10000) + osize += 6; /* \uHHHH */ + else + osize += 10; /* \uHHHHHHHH */ + } + } + + quote = '\''; + if (squote) { + if (dquote) + /* Both squote and dquote present. Use squote, + and escape them */ + osize += squote; + else + quote = '"'; + } + + repr = PyUnicode_New(osize, max); if (repr == NULL) return NULL; - - p = PyUnicode_AS_UNICODE(repr); - - /* Add quote */ - *p++ = (findchar(s, size, '\'') && - !findchar(s, size, '"')) ? '"' : '\''; - while (size-- > 0) { - Py_UNICODE ch = *s++; + okind = PyUnicode_KIND(repr); + odata = PyUnicode_DATA(repr); + + PyUnicode_WRITE(okind, odata, 0, quote); + PyUnicode_WRITE(okind, odata, osize-1, quote); + + for (i = 0, o = 1; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); /* Escape quotes and backslashes */ - if ((ch == PyUnicode_AS_UNICODE(repr)[0]) || (ch == '\\')) { - *p++ = '\\'; - *p++ = ch; + if ((ch == quote) || (ch == '\\')) { + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, ch); continue; } /* Map special whitespace to '\t', \n', '\r' */ if (ch == '\t') { - *p++ = '\\'; - *p++ = 't'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 't'); } else if (ch == '\n') { - *p++ = '\\'; - *p++ = 'n'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'n'); } else if (ch == '\r') { - *p++ = '\\'; - *p++ = 'r'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'r'); } /* Map non-printable US ASCII to '\xhh' */ else if (ch < ' ' || ch == 0x7F) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'x'); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 4) & 0x000F]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[ch & 0x000F]); } /* Copy ASCII characters as-is */ else if (ch < 0x7F) { - *p++ = ch; + PyUnicode_WRITE(okind, odata, o++, ch); } /* Non-ASCII characters */ else { - Py_UCS4 ucs = ch; - -#ifndef Py_UNICODE_WIDE - Py_UNICODE ch2 = 0; - /* Get code point from surrogate pair */ - if (size > 0) { - ch2 = *s; - if (ch >= 0xD800 && ch < 0xDC00 && ch2 >= 0xDC00 - && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) - + 0x00010000; - s++; - size--; - } - } -#endif /* Map Unicode whitespace and control characters (categories Z* and C* except ASCII space) */ - if (!Py_UNICODE_ISPRINTABLE(ucs)) { + if (!Py_UNICODE_ISPRINTABLE(ch)) { /* Map 8-bit characters to '\xhh' */ - if (ucs <= 0xff) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; + if (ch <= 0xff) { + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'x'); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 4) & 0x000F]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[ch & 0x000F]); } /* Map 21-bit characters to '\U00xxxxxx' */ - else if (ucs >= 0x10000) { - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 24) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 20) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 16) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 12) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 8) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 4) & 0x0000000F]; - *p++ = hexdigits[ucs & 0x0000000F]; + else if (ch >= 0x10000) { + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'U'); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 28) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 24) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 20) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 16) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 12) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 8) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 4) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[ch & 0xF]); } /* Map 16-bit characters to '\uxxxx' */ else { - *p++ = '\\'; - *p++ = 'u'; - *p++ = hexdigits[(ucs >> 12) & 0x000F]; - *p++ = hexdigits[(ucs >> 8) & 0x000F]; - *p++ = hexdigits[(ucs >> 4) & 0x000F]; - *p++ = hexdigits[ucs & 0x000F]; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'u'); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 12) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 8) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[(ch >> 4) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, hexdigits[ch & 0xF]); } } /* Copy characters as-is */ else { - *p++ = ch; -#ifndef Py_UNICODE_WIDE - if (ucs >= 0x10000) - *p++ = ch2; -#endif - } - } - } - /* Add quote */ - *p++ = PyUnicode_AS_UNICODE(repr)[0]; - - *p = '\0'; - PyUnicode_Resize(&repr, p - PyUnicode_AS_UNICODE(repr)); + PyUnicode_WRITE(okind, odata, o++, ch); + } + } + } + /* Closing quote already added at the beginning */ return repr; } @@ -8617,7 +10817,7 @@ Return -1 on failure."); static PyObject * -unicode_rfind(PyUnicodeObject *self, PyObject *args) +unicode_rfind(PyObject *self, PyObject *args) { PyUnicodeObject *substring; Py_ssize_t start; @@ -8628,14 +10828,21 @@ &start, &end)) return NULL; - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice( + ucs1lib_rfind_slice, ucs2lib_rfind_slice, ucs4lib_rfind_slice, + self, (PyObject*)substring, start, end ); Py_DECREF(substring); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } @@ -8645,7 +10852,7 @@ Like S.rfind() but raise ValueError when the substring is not found."); static PyObject * -unicode_rindex(PyUnicodeObject *self, PyObject *args) +unicode_rindex(PyObject *self, PyObject *args) { PyUnicodeObject *substring; Py_ssize_t start; @@ -8656,18 +10863,26 @@ &start, &end)) return NULL; - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice( + ucs1lib_rfind_slice, ucs2lib_rfind_slice, ucs4lib_rfind_slice, + self, (PyObject*)substring, start, end ); Py_DECREF(substring); + if (result == -2) + return NULL; + if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; } + return PyLong_FromSsize_t(result); } @@ -8681,17 +10896,20 @@ unicode_rjust(PyUnicodeObject *self, PyObject *args) { Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; + + if (PyUnicode_READY(self) == -1) + return NULL; if (!PyArg_ParseTuple(args, "n|O&:rjust", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { + if (_PyUnicode_LENGTH(self) >= width && PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject*) self; } - return (PyObject*) pad(self, width - self->length, 0, fillchar); + return (PyObject*) pad(self, width - _PyUnicode_LENGTH(self), 0, fillchar); } PyObject * @@ -8749,6 +10967,78 @@ PyObject* str_obj; PyObject* sep_obj; PyObject* out; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; + + str_obj = PyUnicode_FromObject(str_in); + if (!str_obj || PyUnicode_READY(str_in) == -1) + return NULL; + sep_obj = PyUnicode_FromObject(sep_in); + if (!sep_obj || PyUnicode_READY(sep_obj) == -1) { + Py_DECREF(str_obj); + return NULL; + } + + kind1 = PyUnicode_KIND(str_in); + kind2 = PyUnicode_KIND(sep_obj); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(str_in); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(str_in, kind); + if (!buf1) + goto onError; + buf2 = PyUnicode_DATA(sep_obj); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(sep_obj, kind); + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sep_obj); + + switch(PyUnicode_KIND(str_in)) { + case PyUnicode_1BYTE_KIND: + out = ucs1lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + default: + assert(0); + out = 0; + } + + Py_DECREF(sep_obj); + Py_DECREF(str_obj); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + + return out; + onError: + Py_DECREF(sep_obj); + Py_DECREF(str_obj); + if (kind1 != kind && buf1) + PyMem_Free(buf1); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return NULL; +} + + +PyObject * +PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) +{ + PyObject* str_obj; + PyObject* sep_obj; + PyObject* out; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; str_obj = PyUnicode_FromObject(str_in); if (!str_obj) @@ -8759,43 +11049,53 @@ return NULL; } - out = stringlib_partition( - str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), - sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) - ); + kind1 = PyUnicode_KIND(str_in); + kind2 = PyUnicode_KIND(sep_obj); + kind = PY_MAX(kind1, kind2); + buf1 = PyUnicode_DATA(str_in); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(str_in, kind); + if (!buf1) + goto onError; + buf2 = PyUnicode_DATA(sep_obj); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(sep_obj, kind); + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sep_obj); + + switch(PyUnicode_KIND(str_in)) { + case PyUnicode_1BYTE_KIND: + out = ucs1lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + default: + assert(0); + out = 0; + } Py_DECREF(sep_obj); Py_DECREF(str_obj); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); return out; -} - - -PyObject * -PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) -{ - PyObject* str_obj; - PyObject* sep_obj; - PyObject* out; - - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) - return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); - return NULL; - } - - out = stringlib_rpartition( - str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), - sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) - ); - + onError: Py_DECREF(sep_obj); Py_DECREF(str_obj); - - return out; + if (kind1 != kind && buf1) + PyMem_Free(buf1); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return NULL; } PyDoc_STRVAR(partition__doc__, @@ -8943,22 +11243,28 @@ if (!new) return NULL; if (y != NULL) { + int x_kind, y_kind, z_kind; + void *x_data, *y_data, *z_data; + /* x must be a string too, of equal length */ - Py_ssize_t ylen = PyUnicode_GET_SIZE(y); if (!PyUnicode_Check(x)) { PyErr_SetString(PyExc_TypeError, "first maketrans argument must " "be a string if there is a second argument"); goto err; } - if (PyUnicode_GET_SIZE(x) != ylen) { + if (PyUnicode_GET_LENGTH(x) != PyUnicode_GET_LENGTH(y)) { PyErr_SetString(PyExc_ValueError, "the first two maketrans " "arguments must have equal length"); goto err; } /* create entries for translating chars in x to those in y */ - for (i = 0; i < PyUnicode_GET_SIZE(x); i++) { - key = PyLong_FromLong(PyUnicode_AS_UNICODE(x)[i]); - value = PyLong_FromLong(PyUnicode_AS_UNICODE(y)[i]); + x_kind = PyUnicode_KIND(x); + y_kind = PyUnicode_KIND(y); + x_data = PyUnicode_DATA(x); + y_data = PyUnicode_DATA(y); + for (i = 0; i < PyUnicode_GET_LENGTH(x); i++) { + key = PyLong_FromLong(PyUnicode_READ(x_kind, x_data, i)); + value = PyLong_FromLong(PyUnicode_READ(y_kind, y_data, i)); if (!key || !value) goto err; res = PyDict_SetItem(new, key, value); @@ -8969,8 +11275,10 @@ } /* create entries for deleting chars in z */ if (z != NULL) { + z_kind = PyUnicode_KIND(z); + z_data = PyUnicode_DATA(z); for (i = 0; i < PyUnicode_GET_SIZE(z); i++) { - key = PyLong_FromLong(PyUnicode_AS_UNICODE(z)[i]); + key = PyLong_FromLong(PyUnicode_READ(z_kind, z_data, i)); if (!key) goto err; res = PyDict_SetItem(new, key, Py_None); @@ -8980,6 +11288,9 @@ } } } else { + int kind; + void *data; + /* x must be a dict */ if (!PyDict_CheckExact(x)) { PyErr_SetString(PyExc_TypeError, "if you give only one argument " @@ -8996,7 +11307,9 @@ "table must be of length 1"); goto err; } - newkey = PyLong_FromLong(PyUnicode_AS_UNICODE(key)[0]); + kind = PyUnicode_KIND(key); + data = PyUnicode_DATA(key); + newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); if (!newkey) goto err; res = PyDict_SetItem(new, newkey, value); @@ -9030,9 +11343,9 @@ are deleted."); static PyObject* -unicode_translate(PyUnicodeObject *self, PyObject *table) -{ - return PyUnicode_TranslateCharmap(self->str, self->length, table, "ignore"); +unicode_translate(PyObject *self, PyObject *table) +{ + return _PyUnicode_TranslateCharmap(self, table, "ignore"); } PyDoc_STRVAR(upper__doc__, @@ -9057,12 +11370,18 @@ { Py_ssize_t fill; PyUnicodeObject *u; - Py_ssize_t width; + int kind; + void *data; + Py_UCS4 chr; + + if (PyUnicode_READY(self) == -1) + return NULL; + if (!PyArg_ParseTuple(args, "n:zfill", &width)) return NULL; - if (self->length >= width) { + if (PyUnicode_GET_LENGTH(self) >= width) { if (PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject*) self; @@ -9074,34 +11393,31 @@ ); } - fill = width - self->length; + fill = width - _PyUnicode_LENGTH(self); u = pad(self, fill, 0, '0'); if (u == NULL) return NULL; - if (u->str[fill] == '+' || u->str[fill] == '-') { + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); + chr = PyUnicode_READ(kind, data, fill); + + if (chr == '+' || chr == '-') { /* move sign to beginning of string */ - u->str[0] = u->str[fill]; - u->str[fill] = '0'; + PyUnicode_WRITE(kind, data, 0, chr); + PyUnicode_WRITE(kind, data, fill, '0'); } return (PyObject*) u; } #if 0 -static PyObject* -unicode_freelistsize(PyUnicodeObject *self) -{ - return PyLong_FromLong(numfree); -} - static PyObject * unicode__decimal2ascii(PyObject *self) { - return PyUnicode_TransformDecimalToASCII(PyUnicode_AS_UNICODE(self), - PyUnicode_GET_SIZE(self)); + return PyUnicode_TransformDecimalAndSpaceToASCII(self); } #endif @@ -9201,7 +11517,7 @@ return PyBool_FromLong(result); } -#include "stringlib/string_format.h" +#include "stringlib/unicode_format.h" PyDoc_STRVAR(format__doc__, "S.format(*args, **kwargs) -> str\n\ @@ -9223,9 +11539,8 @@ if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyUnicode_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + return _PyUnicode_FormatAdvanced(self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); } PyDoc_STRVAR(p_format__doc__, @@ -9236,8 +11551,35 @@ static PyObject * unicode__sizeof__(PyUnicodeObject *v) { - return PyLong_FromSsize_t(sizeof(PyUnicodeObject) + - sizeof(Py_UNICODE) * (v->length + 1)); + Py_ssize_t size; + + /* If it's a compact object, account for base structure + + character data. */ + if (PyUnicode_IS_COMPACT_ASCII(v)) + size = sizeof(PyASCIIObject) + PyUnicode_GET_LENGTH(v) + 1; + else if (PyUnicode_IS_COMPACT(v)) + size = sizeof(PyCompactUnicodeObject) + + (PyUnicode_GET_LENGTH(v) + 1) * PyUnicode_CHARACTER_SIZE(v); + else { + /* If it is a two-block object, account for base object, and + for character block if present. */ + size = sizeof(PyUnicodeObject); + if (v->data.any) + size += (PyUnicode_GET_LENGTH(v) + 1) * + PyUnicode_CHARACTER_SIZE(v); + } + /* If the wstr pointer is present, account for it unless it is shared + with the data pointer. Since PyUnicode_DATA will crash if the object + is not ready, check whether it's either not ready (in which case the + data is entirely in wstr) or if the data is not shared. */ + if (_PyUnicode_WSTR(v) && + (!PyUnicode_IS_READY(v) || + (PyUnicode_DATA(v) != _PyUnicode_WSTR(v)))) + size += (PyUnicode_WSTR_LENGTH(v) + 1) * sizeof(wchar_t); + if (_PyUnicode_UTF8(v) && _PyUnicode_UTF8(v) != PyUnicode_DATA(v)) + size += _PyUnicode_UTF8_LENGTH(v) + 1; + + return PyLong_FromSsize_t(size); } PyDoc_STRVAR(sizeof__doc__, @@ -9246,7 +11588,17 @@ static PyObject * unicode_getnewargs(PyUnicodeObject *v) { - return Py_BuildValue("(u#)", v->str, v->length); + PyObject *copy; + unsigned char *data; + int kind; + if (PyUnicode_READY(v) == -1) + return NULL; + kind = PyUnicode_KIND(v); + data = PyUnicode_1BYTE_DATA(v); + copy = PyUnicode_FromKindAndData(kind, data, PyUnicode_GET_LENGTH(v)); + if (!copy) + return NULL; + return Py_BuildValue("(N)", copy); } static PyMethodDef unicode_methods[] = { @@ -9306,7 +11658,6 @@ #if 0 /* These methods are just used for debugging the implementation. */ - {"freelistsize", (PyCFunction) unicode_freelistsize, METH_NOARGS}, {"_decimal2ascii", (PyCFunction) unicode__decimal2ascii, METH_NOARGS}, #endif @@ -9343,32 +11694,36 @@ static PyObject* unicode_subscript(PyUnicodeObject* self, PyObject* item) { + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += PyUnicode_GET_SIZE(self); + i += PyUnicode_GET_LENGTH(self); return unicode_getitem(self, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, cur, i; - Py_UNICODE* source_buf; + const Py_UNICODE* source_buf; Py_UNICODE* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_SIZE(self), + if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), &start, &stop, &step, &slicelength) < 0) { return NULL; } if (slicelength <= 0) { - return PyUnicode_FromUnicode(NULL, 0); - } else if (start == 0 && step == 1 && slicelength == self->length && + return PyUnicode_New(0, 0); + } else if (start == 0 && step == 1 && + slicelength == PyUnicode_GET_LENGTH(self) && PyUnicode_CheckExact(self)) { Py_INCREF(self); return (PyObject *)self; } else if (step == 1) { - return PyUnicode_FromUnicode(self->str + start, slicelength); + return substring(self, start, slicelength); } else { source_buf = PyUnicode_AS_UNICODE((PyObject*)self); result_buf = (Py_UNICODE *)PyObject_MALLOC(slicelength* @@ -9436,7 +11791,7 @@ (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL); if (p == NULL) return NULL; - result = PyUnicode_FromStringAndSize(p, strlen(p)); + result = PyUnicode_DecodeASCII(p, strlen(p), NULL); PyMem_Free(p); return result; } @@ -9452,37 +11807,23 @@ str = _PyBytes_FormatLong(val, flags, prec, type, &buf, &len); if (!str) return NULL; - result = PyUnicode_FromStringAndSize(buf, len); + result = PyUnicode_DecodeASCII(buf, len, NULL); Py_DECREF(str); return result; } static int -formatchar(Py_UNICODE *buf, +formatchar(Py_UCS4 *buf, size_t buflen, PyObject *v) { /* presume that the buffer is at least 3 characters long */ if (PyUnicode_Check(v)) { - if (PyUnicode_GET_SIZE(v) == 1) { - buf[0] = PyUnicode_AS_UNICODE(v)[0]; + if (PyUnicode_GET_LENGTH(v) == 1) { + buf[0] = PyUnicode_READ_CHAR(v, 0); buf[1] = '\0'; return 1; } -#ifndef Py_UNICODE_WIDE - if (PyUnicode_GET_SIZE(v) == 2) { - /* Decode a valid surrogate pair */ - int c0 = PyUnicode_AS_UNICODE(v)[0]; - int c1 = PyUnicode_AS_UNICODE(v)[1]; - if (0xD800 <= c0 && c0 <= 0xDBFF && - 0xDC00 <= c1 && c1 <= 0xDFFF) { - buf[0] = c0; - buf[1] = c1; - buf[2] = '\0'; - return 2; - } - } -#endif goto onError; } else { @@ -9498,15 +11839,7 @@ return -1; } -#ifndef Py_UNICODE_WIDE - if (x > 0xffff) { - x -= 0x10000; - buf[0] = (Py_UNICODE)(0xD800 | (x >> 10)); - buf[1] = (Py_UNICODE)(0xDC00 | (x & 0x3FF)); - return 2; - } -#endif - buf[0] = (Py_UNICODE) x; + buf[0] = (Py_UCS4) x; buf[1] = '\0'; return 1; } @@ -9525,28 +11858,35 @@ PyObject * PyUnicode_Format(PyObject *format, PyObject *args) { - Py_UNICODE *fmt, *res; - Py_ssize_t fmtcnt, rescnt, reslen, arglen, argidx; + void *fmt; + int fmtkind; + PyObject *result; + Py_UCS4 *res, *res0; + Py_UCS4 max; + int kind; + Py_ssize_t fmtcnt, fmtpos, rescnt, reslen, arglen, argidx; int args_owned = 0; - PyUnicodeObject *result = NULL; PyObject *dict = NULL; - PyObject *uformat; + PyUnicodeObject *uformat; if (format == NULL || args == NULL) { PyErr_BadInternalCall(); return NULL; } - uformat = PyUnicode_FromObject(format); - if (uformat == NULL) - return NULL; - fmt = PyUnicode_AS_UNICODE(uformat); - fmtcnt = PyUnicode_GET_SIZE(uformat); + uformat = (PyUnicodeObject*)PyUnicode_FromObject(format); + if (uformat == NULL || PyUnicode_READY(uformat) == -1) + return NULL; + fmt = PyUnicode_DATA(uformat); + fmtkind = PyUnicode_KIND(uformat); + fmtcnt = PyUnicode_GET_LENGTH(uformat); + fmtpos = 0; reslen = rescnt = fmtcnt + 100; - result = _PyUnicode_New(reslen); - if (result == NULL) + res = res0 = PyMem_Malloc(reslen * sizeof(Py_UCS4)); + if (res0 == NULL) { + PyErr_NoMemory(); goto onError; - res = PyUnicode_AS_UNICODE(result); + } if (PyTuple_Check(args)) { arglen = PyTuple_Size(args); @@ -9561,35 +11901,39 @@ dict = args; while (--fmtcnt >= 0) { - if (*fmt != '%') { + if (PyUnicode_READ(fmtkind, fmt, fmtpos) != '%') { if (--rescnt < 0) { rescnt = fmtcnt + 100; reslen += rescnt; - if (_PyUnicode_Resize(&result, reslen) < 0) + res0 = PyMem_Realloc(res0, reslen*sizeof(Py_UCS4)); + if (res0 == NULL){ + PyErr_NoMemory(); goto onError; - res = PyUnicode_AS_UNICODE(result) + reslen - rescnt; + } + res = res0 + reslen - rescnt; --rescnt; } - *res++ = *fmt++; + *res++ = PyUnicode_READ(fmtkind, fmt, fmtpos++); } else { /* Got a format specifier */ int flags = 0; Py_ssize_t width = -1; int prec = -1; - Py_UNICODE c = '\0'; - Py_UNICODE fill; + Py_UCS4 c = '\0'; + Py_UCS4 fill; int isnumok; PyObject *v = NULL; PyObject *temp = NULL; - Py_UNICODE *pbuf; + void *pbuf; + Py_ssize_t pindex; Py_UNICODE sign; - Py_ssize_t len; - Py_UNICODE formatbuf[FORMATBUFLEN]; /* For formatchar() */ - - fmt++; - if (*fmt == '(') { - Py_UNICODE *keystart; + Py_ssize_t len, len1; + Py_UCS4 formatbuf[FORMATBUFLEN]; /* For formatchar() */ + + fmtpos++; + if (PyUnicode_READ(fmtkind, fmt, fmtpos) == '(') { + Py_ssize_t keystart; Py_ssize_t keylen; PyObject *key; int pcount = 1; @@ -9599,34 +11943,24 @@ "format requires a mapping"); goto onError; } - ++fmt; + ++fmtpos; --fmtcnt; - keystart = fmt; + keystart = fmtpos; /* Skip over balanced parentheses */ while (pcount > 0 && --fmtcnt >= 0) { - if (*fmt == ')') + if (PyUnicode_READ(fmtkind, fmt, fmtpos) == ')') --pcount; - else if (*fmt == '(') + else if (PyUnicode_READ(fmtkind, fmt, fmtpos) == '(') ++pcount; - fmt++; + fmtpos++; } - keylen = fmt - keystart - 1; + keylen = fmtpos - keystart - 1; if (fmtcnt < 0 || pcount > 0) { PyErr_SetString(PyExc_ValueError, "incomplete format key"); goto onError; } -#if 0 - /* keys are converted to strings using UTF-8 and - then looked up since Python uses strings to hold - variables names etc. in its namespaces and we - wouldn't want to break common idioms. */ - key = PyUnicode_EncodeUTF8(keystart, - keylen, - NULL); -#else - key = PyUnicode_FromUnicode(keystart, keylen); -#endif + key = substring(uformat, keystart, keylen); if (key == NULL) goto onError; if (args_owned) { @@ -9643,7 +11977,7 @@ argidx = -2; } while (--fmtcnt >= 0) { - switch (c = *fmt++) { + switch (c = PyUnicode_READ(fmtkind, fmt, fmtpos++)) { case '-': flags |= F_LJUST; continue; case '+': flags |= F_SIGN; continue; case ' ': flags |= F_BLANK; continue; @@ -9669,12 +12003,12 @@ width = -width; } if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } else if (c >= '0' && c <= '9') { width = c - '0'; while (--fmtcnt >= 0) { - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c < '0' || c > '9') break; if ((width*10) / 10 != width) { @@ -9688,7 +12022,7 @@ if (c == '.') { prec = 0; if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c == '*') { v = getnextarg(args, arglen, &argidx); if (v == NULL) @@ -9704,12 +12038,12 @@ if (prec < 0) prec = 0; if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } else if (c >= '0' && c <= '9') { prec = c - '0'; while (--fmtcnt >= 0) { - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c < '0' || c > '9') break; if ((prec*10) / 10 != prec) { @@ -9724,7 +12058,7 @@ if (fmtcnt >= 0) { if (c == 'h' || c == 'l' || c == 'L') { if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } } if (fmtcnt < 0) { @@ -9743,8 +12077,9 @@ case '%': pbuf = formatbuf; + kind = PyUnicode_4BYTE_KIND; /* presume that buffer length is at least 1 */ - pbuf[0] = '%'; + PyUnicode_WRITE(kind, pbuf, 0, '%'); len = 1; break; @@ -9773,8 +12108,13 @@ goto onError; } } - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); + if (PyUnicode_READY(temp) == -1) { + Py_CLEAR(temp); + goto onError; + } + pbuf = PyUnicode_DATA(temp); + kind = PyUnicode_KIND(temp); + len = PyUnicode_GET_LENGTH(temp); if (prec >= 0 && len > prec) len = prec; break; @@ -9803,8 +12143,13 @@ Py_DECREF(iobj); if (!temp) goto onError; - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); + if (PyUnicode_READY(temp) == -1) { + Py_CLEAR(temp); + goto onError; + } + pbuf = PyUnicode_DATA(temp); + kind = PyUnicode_KIND(temp); + len = PyUnicode_GET_LENGTH(temp); sign = 1; } else { @@ -9831,8 +12176,13 @@ temp = formatfloat(v, flags, prec, c); if (!temp) goto onError; - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); + if (PyUnicode_READY(temp) == -1) { + Py_CLEAR(temp); + goto onError; + } + pbuf = PyUnicode_DATA(temp); + kind = PyUnicode_KIND(temp); + len = PyUnicode_GET_LENGTH(temp); sign = 1; if (flags & F_ZERO) fill = '0'; @@ -9840,6 +12190,7 @@ case 'c': pbuf = formatbuf; + kind = PyUnicode_4BYTE_KIND; len = formatchar(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE), v); if (len < 0) goto onError; @@ -9851,13 +12202,15 @@ "at index %zd", (31<=c && c<=126) ? (char)c : '?', (int)c, - (Py_ssize_t)(fmt - 1 - - PyUnicode_AS_UNICODE(uformat))); + fmtpos - 1); goto onError; } + /* pbuf is initialized here. */ + pindex = 0; if (sign) { - if (*pbuf == '-' || *pbuf == '+') { - sign = *pbuf++; + if (PyUnicode_READ(kind, pbuf, pindex) == '-' || + PyUnicode_READ(kind, pbuf, pindex) == '+') { + sign = PyUnicode_READ(kind, pbuf, pindex++); len--; } else if (flags & F_SIGN) @@ -9878,12 +12231,13 @@ PyErr_NoMemory(); goto onError; } - if (_PyUnicode_Resize(&result, reslen) < 0) { + res0 = PyMem_Realloc(res0, reslen*sizeof(Py_UCS4)); + if (res0 == 0) { + PyErr_NoMemory(); Py_XDECREF(temp); goto onError; } - res = PyUnicode_AS_UNICODE(result) - + reslen - rescnt; + res = res0 + reslen - rescnt; } if (sign) { if (fill != ' ') @@ -9893,11 +12247,11 @@ width--; } if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex+1) == c); if (fill != ' ') { - *res++ = *pbuf++; - *res++ = *pbuf++; + *res++ = PyUnicode_READ(kind, pbuf, pindex++); + *res++ = PyUnicode_READ(kind, pbuf, pindex++); } rescnt -= 2; width -= 2; @@ -9915,15 +12269,18 @@ if (sign) *res++ = sign; if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); - *res++ = *pbuf++; - *res++ = *pbuf++; + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex+1) == c); + *res++ = PyUnicode_READ(kind, pbuf, pindex++); + *res++ = PyUnicode_READ(kind, pbuf, pindex++); } } - Py_UNICODE_COPY(res, pbuf, len); - res += len; - rescnt -= len; + /* Copy all characters, preserving len */ + len1 = len; + while (len1--) { + *res++ = PyUnicode_READ(kind, pbuf, pindex++); + rescnt--; + } while (--width >= len) { --rescnt; *res++ = ' '; @@ -9943,8 +12300,17 @@ goto onError; } - if (_PyUnicode_Resize(&result, reslen - rescnt) < 0) + + for (max=0, res = res0; res < res0+reslen-rescnt; res++) + if (*res > max) + max = *res; + result = PyUnicode_New(reslen - rescnt, max); + if (!result) goto onError; + kind = PyUnicode_KIND(result); + for (res = res0; res < res0+reslen-rescnt; res++) + PyUnicode_WRITE(kind, PyUnicode_DATA(result), res-res0, *res); + PyMem_Free(res0); if (args_owned) { Py_DECREF(args); } @@ -9952,7 +12318,7 @@ return (PyObject *)result; onError: - Py_XDECREF(result); + PyMem_Free(res0); Py_DECREF(uformat); if (args_owned) { Py_DECREF(args); @@ -9977,7 +12343,7 @@ kwlist, &x, &encoding, &errors)) return NULL; if (x == NULL) - return (PyObject *)_PyUnicode_New(0); + return (PyObject *)PyUnicode_New(0, 0); if (encoding == NULL && errors == NULL) return PyObject_Str(x); else @@ -9989,29 +12355,54 @@ { PyUnicodeObject *tmp, *pnew; Py_ssize_t n; + PyObject *err = NULL; assert(PyType_IsSubtype(type, &PyUnicode_Type)); tmp = (PyUnicodeObject *)unicode_new(&PyUnicode_Type, args, kwds); if (tmp == NULL) return NULL; assert(PyUnicode_Check(tmp)); - pnew = (PyUnicodeObject *) type->tp_alloc(type, n = tmp->length); + // TODO: Verify the PyUnicode_GET_SIZE does the right thing. + // it seems kind of strange that tp_alloc gets passed the size + // of the unicode string because there will follow another + // malloc. + pnew = (PyUnicodeObject *) type->tp_alloc(type, + n = PyUnicode_GET_SIZE(tmp)); if (pnew == NULL) { Py_DECREF(tmp); return NULL; } - pnew->str = (Py_UNICODE*) PyObject_MALLOC(sizeof(Py_UNICODE) * (n+1)); - if (pnew->str == NULL) { - _Py_ForgetReference((PyObject *)pnew); - PyObject_Del(pnew); - Py_DECREF(tmp); - return PyErr_NoMemory(); - } - Py_UNICODE_COPY(pnew->str, tmp->str, n+1); - pnew->length = n; - pnew->hash = tmp->hash; + _PyUnicode_WSTR(pnew) = (Py_UNICODE*) PyObject_MALLOC(sizeof(Py_UNICODE) * (n+1)); + if (_PyUnicode_WSTR(pnew) == NULL) { + err = PyErr_NoMemory(); + goto onError; + } + Py_UNICODE_COPY(_PyUnicode_WSTR(pnew), PyUnicode_AS_UNICODE(tmp), n+1); + _PyUnicode_WSTR_LENGTH(pnew) = n; + _PyUnicode_HASH(pnew) = _PyUnicode_HASH(tmp); + _PyUnicode_STATE(pnew).interned = 0; + _PyUnicode_STATE(pnew).kind = 0; + _PyUnicode_STATE(pnew).compact = 0; + _PyUnicode_STATE(pnew).ready = 0; + _PyUnicode_STATE(pnew).ascii = 0; + pnew->data.any = NULL; + _PyUnicode_LENGTH(pnew) = 0; + pnew->_base.utf8 = NULL; + pnew->_base.utf8_length = 0; + + if (PyUnicode_READY(pnew) == -1) { + PyObject_FREE(_PyUnicode_WSTR(pnew)); + goto onError; + } + Py_DECREF(tmp); return (PyObject *)pnew; + + onError: + _Py_ForgetReference((PyObject *)pnew); + PyObject_Del(pnew); + Py_DECREF(tmp); + return err; } PyDoc_STRVAR(unicode_doc, @@ -10074,7 +12465,7 @@ int i; /* XXX - move this array to unicodectype.c ? */ - Py_UNICODE linebreak[] = { + Py_UCS2 linebreak[] = { 0x000A, /* LINE FEED */ 0x000D, /* CARRIAGE RETURN */ 0x001C, /* FILE SEPARATOR */ @@ -10086,11 +12477,9 @@ }; /* Init the implementation */ - free_list = NULL; - numfree = 0; - unicode_empty = _PyUnicode_New(0); + unicode_empty = (PyUnicodeObject *) PyUnicode_New(0, 0); if (!unicode_empty) - return; + Py_FatalError("Can't create empty string"); for (i = 0; i < 256; i++) unicode_latin1[i] = NULL; @@ -10099,8 +12488,8 @@ /* initialize the linebreak bloom filter */ bloom_linebreak = make_bloom_mask( - linebreak, sizeof(linebreak) / sizeof(linebreak[0]) - ); + PyUnicode_2BYTE_KIND, linebreak, + sizeof(linebreak) / sizeof(linebreak[0])); PyType_Ready(&EncodingMapType); } @@ -10110,21 +12499,7 @@ int PyUnicode_ClearFreeList(void) { - int freelist_size = numfree; - PyUnicodeObject *u; - - for (u = free_list; u != NULL;) { - PyUnicodeObject *v = u; - u = *(PyUnicodeObject **)u; - if (v->str) - PyObject_DEL(v->str); - Py_XDECREF(v->defenc); - PyObject_Del(v); - numfree--; - } - free_list = NULL; - assert(numfree == 0); - return freelist_size; + return 0; } void @@ -10158,6 +12533,10 @@ return; if (PyUnicode_CHECK_INTERNED(s)) return; + if (PyUnicode_READY(s) == -1) { + assert(0 && "ready fail in intern..."); + return; + } if (interned == NULL) { interned = PyDict_New(); if (interned == NULL) { @@ -10189,15 +12568,17 @@ /* The two references in interned are not counted by refcnt. The deallocator will take care of this */ Py_REFCNT(s) -= 2; - PyUnicode_CHECK_INTERNED(s) = SSTATE_INTERNED_MORTAL; + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; } void PyUnicode_InternImmortal(PyObject **p) { + PyUnicodeObject *u = (PyUnicodeObject *)*p; + PyUnicode_InternInPlace(p); if (PyUnicode_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) { - PyUnicode_CHECK_INTERNED(*p) = SSTATE_INTERNED_IMMORTAL; + _PyUnicode_STATE(u).interned = SSTATE_INTERNED_IMMORTAL; Py_INCREF(*p); } } @@ -10238,22 +12619,24 @@ n); for (i = 0; i < n; i++) { s = (PyUnicodeObject *) PyList_GET_ITEM(keys, i); - switch (s->state) { + if (PyUnicode_READY(s) == -1) + fprintf(stderr, "could not ready string\n"); + switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_NOT_INTERNED: /* XXX Shouldn't happen */ break; case SSTATE_INTERNED_IMMORTAL: Py_REFCNT(s) += 1; - immortal_size += s->length; + immortal_size += PyUnicode_GET_LENGTH(s); break; case SSTATE_INTERNED_MORTAL: Py_REFCNT(s) += 2; - mortal_size += s->length; + mortal_size += PyUnicode_GET_LENGTH(s); break; default: Py_FatalError("Inconsistent interned string state."); } - s->state = SSTATE_NOT_INTERNED; + _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; } fprintf(stderr, "total size of all interned strings: " "%" PY_FORMAT_SIZE_T "d/%" PY_FORMAT_SIZE_T "d " @@ -10300,9 +12683,11 @@ return NULL; assert(PyUnicode_Check(seq)); - if (it->it_index < PyUnicode_GET_SIZE(seq)) { - item = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(seq)+it->it_index, 1); + if (it->it_index < PyUnicode_GET_LENGTH(seq)) { + int kind = PyUnicode_KIND(seq); + void *data = PyUnicode_DATA(seq); + Py_UCS4 chr = PyUnicode_READ(kind, data, it->it_index); + item = PyUnicode_FromOrdinal(chr); if (item != NULL) ++it->it_index; return item; @@ -10372,6 +12757,8 @@ PyErr_BadInternalCall(); return NULL; } + if (PyUnicode_READY(seq) == -1) + return NULL; it = PyObject_GC_New(unicodeiterobject, &PyUnicodeIter_Type); if (it == NULL) return NULL; @@ -10382,95 +12769,16 @@ return (PyObject *)it; } -size_t -Py_UNICODE_strlen(const Py_UNICODE *u) -{ - int res = 0; - while(*u++) - res++; - return res; -} - -Py_UNICODE* -Py_UNICODE_strcpy(Py_UNICODE *s1, const Py_UNICODE *s2) -{ - Py_UNICODE *u = s1; - while ((*u++ = *s2++)); - return s1; -} - -Py_UNICODE* -Py_UNICODE_strncpy(Py_UNICODE *s1, const Py_UNICODE *s2, size_t n) -{ - Py_UNICODE *u = s1; - while ((*u++ = *s2++)) - if (n-- == 0) - break; - return s1; -} - -Py_UNICODE* -Py_UNICODE_strcat(Py_UNICODE *s1, const Py_UNICODE *s2) -{ - Py_UNICODE *u1 = s1; - u1 += Py_UNICODE_strlen(u1); - Py_UNICODE_strcpy(u1, s2); - return s1; -} - -int -Py_UNICODE_strcmp(const Py_UNICODE *s1, const Py_UNICODE *s2) -{ - while (*s1 && *s2 && *s1 == *s2) - s1++, s2++; - if (*s1 && *s2) - return (*s1 < *s2) ? -1 : +1; - if (*s1) - return 1; - if (*s2) - return -1; - return 0; -} - -int -Py_UNICODE_strncmp(const Py_UNICODE *s1, const Py_UNICODE *s2, size_t n) -{ - register Py_UNICODE u1, u2; - for (; n != 0; n--) { - u1 = *s1; - u2 = *s2; - if (u1 != u2) - return (u1 < u2) ? -1 : +1; - if (u1 == '\0') - return 0; - s1++; - s2++; - } - return 0; -} - -Py_UNICODE* -Py_UNICODE_strchr(const Py_UNICODE *s, Py_UNICODE c) -{ - const Py_UNICODE *p; - for (p = s; *p; p++) - if (*p == c) - return (Py_UNICODE*)p; - return NULL; -} - -Py_UNICODE* -Py_UNICODE_strrchr(const Py_UNICODE *s, Py_UNICODE c) -{ - const Py_UNICODE *p; - p = s + Py_UNICODE_strlen(s); - while (p != s) { - p--; - if (*p == c) - return (Py_UNICODE*)p; - } - return NULL; -} +#define UNIOP(x) Py_UNICODE_##x +#define UNIOP_t Py_UNICODE +#include "uniops.h" +#undef UNIOP +#undef UNIOP_t +#define UNIOP(x) Py_UCS4_##x +#define UNIOP_t Py_UCS4 +#include "uniops.h" +#undef UNIOP +#undef UNIOP_t Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *object) @@ -10479,6 +12787,10 @@ Py_UNICODE *copy; Py_ssize_t size; + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } /* Ensure we won't overflow the size. */ if (PyUnicode_GET_SIZE(unicode) > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { PyErr_NoMemory(); diff --git a/Objects/uniops.h b/Objects/uniops.h new file mode 100644 --- /dev/null +++ b/Objects/uniops.h @@ -0,0 +1,91 @@ + +size_t +UNIOP(strlen)(const UNIOP_t *u) +{ + int res = 0; + while(*u++) + res++; + return res; +} + +UNIOP_t* +UNIOP(strcpy)(UNIOP_t *s1, const UNIOP_t *s2) +{ + UNIOP_t *u = s1; + while ((*u++ = *s2++)); + return s1; +} + +UNIOP_t* +UNIOP(strncpy)(UNIOP_t *s1, const UNIOP_t *s2, size_t n) +{ + UNIOP_t *u = s1; + while ((*u++ = *s2++)) + if (n-- == 0) + break; + return s1; +} + +UNIOP_t* +UNIOP(strcat)(UNIOP_t *s1, const UNIOP_t *s2) +{ + UNIOP_t *u1 = s1; + u1 += UNIOP(strlen(u1)); + UNIOP(strcpy(u1, s2)); + return s1; +} + +int +UNIOP(strcmp)(const UNIOP_t *s1, const UNIOP_t *s2) +{ + while (*s1 && *s2 && *s1 == *s2) + s1++, s2++; + if (*s1 && *s2) + return (*s1 < *s2) ? -1 : +1; + if (*s1) + return 1; + if (*s2) + return -1; + return 0; +} + +int +UNIOP(strncmp)(const UNIOP_t *s1, const UNIOP_t *s2, size_t n) +{ + register UNIOP_t u1, u2; + for (; n != 0; n--) { + u1 = *s1; + u2 = *s2; + if (u1 != u2) + return (u1 < u2) ? -1 : +1; + if (u1 == '\0') + return 0; + s1++; + s2++; + } + return 0; +} + +UNIOP_t* +UNIOP(strchr)(const UNIOP_t *s, UNIOP_t c) +{ + const UNIOP_t *p; + for (p = s; *p; p++) + if (*p == c) + return (UNIOP_t*)p; + return NULL; +} + +UNIOP_t* +UNIOP(strrchr)(const UNIOP_t *s, UNIOP_t c) +{ + const UNIOP_t *p; + p = s + UNIOP(strlen)(s); + while (p != s) { + p--; + if (*p == c) + return (UNIOP_t*)p; + } + return NULL; +} + diff --git a/PC/_subprocess.c b/PC/_subprocess.c --- a/PC/_subprocess.c +++ b/PC/_subprocess.c @@ -329,12 +329,9 @@ static PyObject* getenvironment(PyObject* environment) { - int i; - Py_ssize_t envsize; - PyObject* out = NULL; - PyObject* keys; - PyObject* values; - Py_UNICODE* p; + Py_ssize_t i, envsize, totalsize; + Py_UCS4 *buffer = NULL, *p, *end; + PyObject *keys, *values, *res; /* convert environment dictionary to windows enviroment string */ if (! PyMapping_Check(environment)) { @@ -350,14 +347,8 @@ if (!keys || !values) goto error; - out = PyUnicode_FromUnicode(NULL, 2048); - if (! out) - goto error; - - p = PyUnicode_AS_UNICODE(out); - + totalsize = 1; /* trailing null character */ for (i = 0; i < envsize; i++) { - Py_ssize_t ksize, vsize, totalsize; PyObject* key = PyList_GET_ITEM(keys, i); PyObject* value = PyList_GET_ITEM(values, i); @@ -366,36 +357,42 @@ "environment can only contain strings"); goto error; } - ksize = PyUnicode_GET_SIZE(key); - vsize = PyUnicode_GET_SIZE(value); - totalsize = (p - PyUnicode_AS_UNICODE(out)) + ksize + 1 + - vsize + 1 + 1; - if (totalsize > PyUnicode_GET_SIZE(out)) { - Py_ssize_t offset = p - PyUnicode_AS_UNICODE(out); - PyUnicode_Resize(&out, totalsize + 1024); - p = PyUnicode_AS_UNICODE(out) + offset; - } - Py_UNICODE_COPY(p, PyUnicode_AS_UNICODE(key), ksize); - p += ksize; + totalsize += PyUnicode_GET_LENGTH(key) + 1; /* +1 for '=' */ + totalsize += PyUnicode_GET_LENGTH(value) + 1; /* +1 for '\0' */ + } + + buffer = PyMem_Malloc(totalsize * sizeof(Py_UCS4)); + if (! buffer) + goto error; + p = buffer; + end = buffer + totalsize; + + for (i = 0; i < envsize; i++) { + PyObject* key = PyList_GET_ITEM(keys, i); + PyObject* value = PyList_GET_ITEM(values, i); + if (!PyUnicode_AsUCS4(key, p, end - p, 0)) + goto error; + p += PyUnicode_GET_LENGTH(key); *p++ = '='; - Py_UNICODE_COPY(p, PyUnicode_AS_UNICODE(value), vsize); - p += vsize; + if (!PyUnicode_AsUCS4(value, p, end - p, 0)) + goto error; + p += PyUnicode_GET_LENGTH(value); *p++ = '\0'; } /* add trailing null byte */ *p++ = '\0'; - PyUnicode_Resize(&out, p - PyUnicode_AS_UNICODE(out)); - - /* PyObject_Print(out, stdout, 0); */ + assert(p == end); Py_XDECREF(keys); Py_XDECREF(values); - return out; + res = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, p - buffer); + PyMem_Free(buffer); + return res; error: - Py_XDECREF(out); + PyMem_Free(buffer); Py_XDECREF(keys); Py_XDECREF(values); return NULL; @@ -609,7 +606,7 @@ if (! result) return PyErr_SetFromWindowsErr(GetLastError()); - return PyUnicode_FromUnicode(filename, Py_UNICODE_strlen(filename)); + return PyUnicode_FromWideChar(filename, wcslen(filename)); } static PyMethodDef sp_functions[] = { diff --git a/PC/import_nt.c b/PC/import_nt.c --- a/PC/import_nt.c +++ b/PC/import_nt.c @@ -93,7 +93,7 @@ } if (fdp->suffix == NULL) return NULL; - path = PyUnicode_FromUnicode(pathBuf, wcslen(pathBuf)); + path = PyUnicode_FromWideChar(pathBuf, wcslen(pathBuf)); if (path == NULL) return NULL; fp = _Py_fopen(path, fdp->mode); diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -212,7 +212,6 @@ msvcrt_getwch(PyObject *self, PyObject *args) { Py_UNICODE ch; - Py_UNICODE u[1]; if (!PyArg_ParseTuple(args, ":getwch")) return NULL; @@ -220,8 +219,7 @@ Py_BEGIN_ALLOW_THREADS ch = _getwch(); Py_END_ALLOW_THREADS - u[0] = ch; - return PyUnicode_FromUnicode(u, 1); + return PyUnicode_FromOrdinal(ch); } PyDoc_STRVAR(getwch_doc, @@ -257,7 +255,6 @@ msvcrt_getwche(PyObject *self, PyObject *args) { Py_UNICODE ch; - Py_UNICODE s[1]; if (!PyArg_ParseTuple(args, ":getwche")) return NULL; @@ -265,8 +262,7 @@ Py_BEGIN_ALLOW_THREADS ch = _getwche(); Py_END_ALLOW_THREADS - s[0] = ch; - return PyUnicode_FromUnicode(s, 1); + return PyUnicode_FromOrdinal(ch); } PyDoc_STRVAR(getwche_doc, diff --git a/PC/pyconfig.h b/PC/pyconfig.h --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -550,10 +550,6 @@ /* Define if you want to use the GNU readline library */ /* #define WITH_READLINE 1 */ -/* Define as the size of the unicode type. */ -/* This is enough for unicodeobject.h to do the "right thing" on Windows. */ -#define Py_UNICODE_SIZE 2 - /* Use Python's own small-block memory-allocator. */ #define WITH_PYMALLOC 1 diff --git a/PC/winreg.c b/PC/winreg.c --- a/PC/winreg.c +++ b/PC/winreg.c @@ -882,7 +882,7 @@ retDataSize -= 2; if (retDataSize <= 0) data = L""; - obData = PyUnicode_FromUnicode(data, retDataSize/2); + obData = PyUnicode_FromWideChar(data, retDataSize/2); break; } case REG_MULTI_SZ: @@ -913,7 +913,7 @@ } PyList_SetItem(obData, index, - PyUnicode_FromUnicode(str[index], len)); + PyUnicode_FromWideChar(str[index], len)); } free(str); @@ -1123,7 +1123,7 @@ if (rc != ERROR_SUCCESS) return PyErr_SetFromWindowsErrWithFunction(rc, "RegEnumKeyEx"); - retStr = PyUnicode_FromUnicode(tmpbuf, len); + retStr = PyUnicode_FromWideChar(tmpbuf, len); return retStr; /* can be NULL */ } @@ -1394,7 +1394,7 @@ "RegQueryValue"); } - retStr = PyUnicode_FromUnicode(retBuf, wcslen(retBuf)); + retStr = PyUnicode_FromWideChar(retBuf, wcslen(retBuf)); PyMem_Free(retBuf); return retStr; } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1258,14 +1258,16 @@ #ifdef PGEN #define verify_identifier(tok) 1 #else -/* Verify that the identifier follows PEP 3131. */ +/* Verify that the identifier follows PEP 3131. + All identifier strings are guaranteed to be "ready" unicode objects. + */ static int verify_identifier(struct tok_state *tok) { PyObject *s; int result; s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); - if (s == NULL) { + if (s == NULL || PyUnicode_READY(s) == -1) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { PyErr_Clear(); tok->done = E_IDENTIFIER; diff --git a/Python/_warnings.c b/Python/_warnings.c --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -498,17 +498,19 @@ *filename = PyDict_GetItemString(globals, "__file__"); if (*filename != NULL && PyUnicode_Check(*filename)) { Py_ssize_t len = PyUnicode_GetSize(*filename); - Py_UNICODE *unicode = PyUnicode_AS_UNICODE(*filename); + int kind = PyUnicode_KIND(*filename); + void *data = PyUnicode_DATA(*filename); /* if filename.lower().endswith((".pyc", ".pyo")): */ if (len >= 4 && - unicode[len-4] == '.' && - Py_UNICODE_TOLOWER(unicode[len-3]) == 'p' && - Py_UNICODE_TOLOWER(unicode[len-2]) == 'y' && - (Py_UNICODE_TOLOWER(unicode[len-1]) == 'c' || - Py_UNICODE_TOLOWER(unicode[len-1]) == 'o')) + PyUnicode_READ(kind, data, len-4) == '.' && + Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-3)) == 'p' && + Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-2)) == 'y' && + (Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-1)) == 'c' || + Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-1)) == 'o')) { - *filename = PyUnicode_FromUnicode(unicode, len-1); + *filename = PyUnicode_Substring(*filename, 0, + PyUnicode_GET_LENGTH(*filename)-1); if (*filename == NULL) goto handle_error; } diff --git a/Python/ast.c b/Python/ast.c --- a/Python/ast.c +++ b/Python/ast.c @@ -528,26 +528,21 @@ new_identifier(const char* n, PyArena *arena) { PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); - Py_UNICODE *u; - if (!id) + if (!id || PyUnicode_READY(id) == -1) return NULL; - u = PyUnicode_AS_UNICODE(id); /* Check whether there are non-ASCII characters in the identifier; if so, normalize to NFKC. */ - for (; *u; u++) { - if (*u >= 128) { - PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); - PyObject *id2; - if (!m) - return NULL; - id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id); - Py_DECREF(m); - if (!id2) - return NULL; - Py_DECREF(id); - id = id2; - break; - } + if (PyUnicode_MAX_CHAR_VALUE((PyUnicodeObject *)id) >= 128) { + PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); + PyObject *id2; + if (!m) + return NULL; + id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id); + Py_DECREF(m); + if (!id2) + return NULL; + Py_DECREF(id); + id = id2; } PyUnicode_InternInPlace(&id); PyArena_AddPyObject(arena, id); @@ -3660,20 +3655,14 @@ } static PyObject * -decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encoding) +decode_utf8(struct compiling *c, const char **sPtr, const char *end) { - PyObject *u, *v; char *s, *t; t = s = (char *)*sPtr; /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ while (s < end && (*s & 0x80)) s++; *sPtr = s; - u = PyUnicode_DecodeUTF8(t, s - t, NULL); - if (u == NULL) - return NULL; - v = PyUnicode_AsEncodedString(u, encoding, NULL); - Py_DECREF(u); - return v; + return PyUnicode_DecodeUTF8(t, s - t, NULL); } static PyObject * @@ -3707,22 +3696,20 @@ } if (*s & 0x80) { /* XXX inefficient */ PyObject *w; - char *r; - Py_ssize_t rn, i; - w = decode_utf8(c, &s, end, "utf-32-be"); + int kind; + void *data; + Py_ssize_t len, i; + w = decode_utf8(c, &s, end); if (w == NULL) { Py_DECREF(u); return NULL; } - r = PyBytes_AS_STRING(w); - rn = Py_SIZE(w); - assert(rn % 4 == 0); - for (i = 0; i < rn; i += 4) { - sprintf(p, "\\U%02x%02x%02x%02x", - r[i + 0] & 0xFF, - r[i + 1] & 0xFF, - r[i + 2] & 0xFF, - r[i + 3] & 0xFF); + kind = PyUnicode_KIND(w); + data = PyUnicode_DATA(w); + len = PyUnicode_GET_LENGTH(w); + for (i = 0; i < len; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + sprintf(p, "\\U%08x", chr); p += 10; } /* Should be impossible to overflow */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -508,8 +508,8 @@ if (PyUnicode_Check(cmd)) { cf->cf_flags |= PyCF_IGNORE_COOKIE; - cmd = _PyUnicode_AsDefaultEncodedString(cmd); - if (cmd == NULL) + str = PyUnicode_AsUTF8AndSize(cmd, &size); + if (str == NULL) return NULL; } else if (!PyObject_CheckReadBuffer(cmd)) { @@ -518,9 +518,10 @@ funcname, what); return NULL; } - if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) { + else if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) { return NULL; } + if (strlen(str) != size) { PyErr_SetString(PyExc_TypeError, "source code string cannot contain null bytes"); @@ -1395,24 +1396,13 @@ } } else if (PyUnicode_Check(obj)) { - size = PyUnicode_GET_SIZE(obj); + if (PyUnicode_READY(obj) == -1) + return NULL; + size = PyUnicode_GET_LENGTH(obj); if (size == 1) { - ord = (long)*PyUnicode_AS_UNICODE(obj); + ord = (long)PyUnicode_READ_CHAR(obj, 0); return PyLong_FromLong(ord); } -#ifndef Py_UNICODE_WIDE - if (size == 2) { - /* Decode a valid surrogate pair */ - int c0 = PyUnicode_AS_UNICODE(obj)[0]; - int c1 = PyUnicode_AS_UNICODE(obj)[1]; - if (0xD800 <= c0 && c0 <= 0xDBFF && - 0xDC00 <= c1 && c1 <= 0xDFFF) { - ord = ((((c0 & 0x03FF) << 10) | (c1 & 0x03FF)) + - 0x00010000); - return PyLong_FromLong(ord); - } - } -#endif } else if (PyByteArray_Check(obj)) { /* XXX Hopefully this is temporary */ diff --git a/Python/ceval.c b/Python/ceval.c --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2054,7 +2054,7 @@ /* Inline the PyDict_GetItem() calls. WARNING: this is an extreme speed hack. Do not try this at home. */ - Py_hash_t hash = ((PyUnicodeObject *)w)->hash; + Py_hash_t hash = ((PyASCIIObject *)w)->hash; if (hash != -1) { PyDictObject *d; PyDictEntry *e; @@ -4456,7 +4456,8 @@ } if (skip_leading_underscores && PyUnicode_Check(name) && - PyUnicode_AS_UNICODE(name)[0] == '_') + PyUnicode_READY(name) != -1 && + PyUnicode_READ_CHAR(name, 0) == '_') { Py_DECREF(name); continue; @@ -4520,6 +4521,14 @@ { /* This function implements 'variable += expr' when both arguments are (Unicode) strings. */ + + w = PyUnicode_Concat(v, w); + Py_DECREF(v); + return w; + + /* XXX: This optimization is currently disabled as unicode objects in the + new flexible representation are not in-place resizable anymore. */ +#if 0 Py_ssize_t v_len = PyUnicode_GET_SIZE(v); Py_ssize_t w_len = PyUnicode_GET_SIZE(w); Py_ssize_t new_len = v_len + w_len; @@ -4570,7 +4579,8 @@ } } - if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v)) { + if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v) && + !PyUnicode_IS_COMPACT((PyUnicodeObject *)v)) { /* Now we own the last reference to 'v', so we can resize it * in-place. */ @@ -4594,6 +4604,7 @@ Py_DECREF(v); return w; } +#endif } #ifdef DYNAMIC_EXECUTION_PROFILE diff --git a/Python/codecs.c b/Python/codecs.c --- a/Python/codecs.c +++ b/Python/codecs.c @@ -513,27 +513,25 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc) { - PyObject *restuple; - Py_ssize_t start; - Py_ssize_t end; - Py_ssize_t i; + Py_ssize_t start, end, i, len; if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { PyObject *res; - Py_UNICODE *p; + int kind; + void *data; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; if (PyUnicodeEncodeError_GetEnd(exc, &end)) return NULL; - res = PyUnicode_FromUnicode(NULL, end-start); + len = end - start; + res = PyUnicode_New(len, '?'); if (res == NULL) return NULL; - for (p = PyUnicode_AS_UNICODE(res), i = start; - i<end; ++p, ++i) - *p = '?'; - restuple = Py_BuildValue("(On)", res, end); - Py_DECREF(res); - return restuple; + kind = PyUnicode_KIND(res); + data = PyUnicode_DATA(res); + for (i = 0; i < len; ++i) + PyUnicode_WRITE(kind, data, i, '?'); + return Py_BuildValue("(Nn)", res, end); } else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) { Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER; @@ -543,20 +541,21 @@ } else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) { PyObject *res; - Py_UNICODE *p; + int kind; + void *data; if (PyUnicodeTranslateError_GetStart(exc, &start)) return NULL; if (PyUnicodeTranslateError_GetEnd(exc, &end)) return NULL; - res = PyUnicode_FromUnicode(NULL, end-start); + len = end - start; + res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER); if (res == NULL) return NULL; - for (p = PyUnicode_AS_UNICODE(res), i = start; - i<end; ++p, ++i) - *p = Py_UNICODE_REPLACEMENT_CHARACTER; - restuple = Py_BuildValue("(On)", res, end); - Py_DECREF(res); - return restuple; + kind = PyUnicode_KIND(res); + data = PyUnicode_DATA(res); + for (i=0; i < len; i++) + PyUnicode_WRITE(kind, data, i, Py_UNICODE_REPLACEMENT_CHARACTER); + return Py_BuildValue("(Nn)", res, end); } else { wrong_exception_type(exc); @@ -671,10 +670,7 @@ } } -static Py_UNICODE hexdigits[] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' -}; +static const char *hexdigits = "0123456789abcdef"; PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) { diff --git a/Python/compile.c b/Python/compile.c --- a/Python/compile.c +++ b/Python/compile.c @@ -197,16 +197,17 @@ { /* Name mangling: __private becomes _classname__private. This is independent from how the name is used. */ - const Py_UNICODE *p, *name = PyUnicode_AS_UNICODE(ident); - Py_UNICODE *buffer; - size_t nlen, plen; + PyObject *result; + size_t nlen, plen, ipriv; + Py_UCS4 maxchar; if (privateobj == NULL || !PyUnicode_Check(privateobj) || - name == NULL || name[0] != '_' || name[1] != '_') { + PyUnicode_READ_CHAR(ident, 0) != '_' || + PyUnicode_READ_CHAR(ident, 1) != '_') { Py_INCREF(ident); return ident; } - p = PyUnicode_AS_UNICODE(privateobj); - nlen = Py_UNICODE_strlen(name); + nlen = PyUnicode_GET_LENGTH(ident); + plen = PyUnicode_GET_LENGTH(privateobj); /* Don't mangle __id__ or names with dots. The only time a name with a dot can occur is when @@ -216,32 +217,37 @@ TODO(jhylton): Decide whether we want to support mangling of the module name, e.g. __M.X. */ - if ((name[nlen-1] == '_' && name[nlen-2] == '_') - || Py_UNICODE_strchr(name, '.')) { + if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' && + PyUnicode_READ_CHAR(ident, nlen-2) == '_') || + PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) { Py_INCREF(ident); return ident; /* Don't mangle __whatever__ */ } /* Strip leading underscores from class name */ - while (*p == '_') - p++; - if (*p == 0) { + ipriv = 0; + while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_') + ipriv++; + if (ipriv == plen) { Py_INCREF(ident); return ident; /* Don't mangle if class is just underscores */ } - plen = Py_UNICODE_strlen(p); + plen -= ipriv; assert(1 <= PY_SSIZE_T_MAX - nlen); assert(1 + nlen <= PY_SSIZE_T_MAX - plen); - ident = PyUnicode_FromStringAndSize(NULL, 1 + nlen + plen); - if (!ident) + maxchar = PyUnicode_MAX_CHAR_VALUE(ident); + if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar) + maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj); + + result = PyUnicode_New(1 + nlen + plen, maxchar); + if (!result) return 0; - /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */ - buffer = PyUnicode_AS_UNICODE(ident); - buffer[0] = '_'; - Py_UNICODE_strncpy(buffer+1, p, plen); - Py_UNICODE_strcpy(buffer+1+plen, name); - return ident; + /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ + PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_'); + PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen); + PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen); + return result; } static int @@ -2085,22 +2091,27 @@ If there is a dot in name, we need to split it and emit a LOAD_ATTR for each name. */ - const Py_UNICODE *src = PyUnicode_AS_UNICODE(name); - const Py_UNICODE *dot = Py_UNICODE_strchr(src, '.'); - if (dot) { + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) + return -1; + if (dot != -1) { /* Consume the base module name to get the first attribute */ - src = dot + 1; - while (dot) { - /* NB src is only defined when dot != NULL */ + Py_ssize_t pos = dot + 1; + while (dot != -1) { PyObject *attr; - dot = Py_UNICODE_strchr(src, '.'); - attr = PyUnicode_FromUnicode(src, - dot ? dot - src : Py_UNICODE_strlen(src)); + dot = PyUnicode_FindChar(name, '.', pos, + PyUnicode_GET_LENGTH(name), 1); + if (dot == -2) + return -1; + attr = PyUnicode_Substring(name, pos, + (dot != -1) ? dot : + PyUnicode_GET_LENGTH(name)); if (!attr) return -1; ADDOP_O(c, LOAD_ATTR, attr, names); Py_DECREF(attr); - src = dot + 1; + pos = dot + 1; } } return compiler_nameop(c, asname, Store); @@ -2139,13 +2150,12 @@ } else { identifier tmp = alias->name; - const Py_UNICODE *base = PyUnicode_AS_UNICODE(alias->name); - Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); - if (dot) - tmp = PyUnicode_FromUnicode(base, - dot - base); + Py_ssize_t dot = PyUnicode_FindChar( + alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); + if (dot != -1) + tmp = PyUnicode_Substring(alias->name, 0, dot); r = compiler_nameop(c, tmp, Store); - if (dot) { + if (dot != -1) { Py_DECREF(tmp); } if (!r) @@ -2208,7 +2218,7 @@ alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); identifier store_name; - if (i == 0 && *PyUnicode_AS_UNICODE(alias->name) == '*') { + if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { assert(n == 1); ADDOP(c, IMPORT_STAR); return 1; @@ -2522,7 +2532,7 @@ } /* XXX Leave assert here, but handle __doc__ and the like better */ - assert(scope || PyUnicode_AS_UNICODE(name)[0] == '_'); + assert(scope || PyUnicode_READ_CHAR(name, 0) == '_'); switch (optype) { case OP_DEREF: @@ -3045,8 +3055,7 @@ return PyObject_IsTrue(e->v.Str.s); case Name_kind: /* optimize away names that can't be reassigned */ - id = PyBytes_AS_STRING( - _PyUnicode_AsDefaultEncodedString(e->v.Name.id)); + id = PyUnicode_AsUTF8(e->v.Name.id); if (strcmp(id, "True") == 0) return 1; if (strcmp(id, "False") == 0) return 0; if (strcmp(id, "None") == 0) return 0; diff --git a/Python/errors.c b/Python/errors.c --- a/Python/errors.c +++ b/Python/errors.c @@ -395,7 +395,7 @@ /* remove trailing cr/lf and dots */ while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) s_buf[--len] = L'\0'; - message = PyUnicode_FromUnicode(s_buf, len); + message = PyUnicode_FromWideChar(s_buf, len); } } } @@ -487,7 +487,7 @@ /* remove trailing cr/lf and dots */ while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.')) s_buf[--len] = L'\0'; - message = PyUnicode_FromUnicode(s_buf, len); + message = PyUnicode_FromWideChar(s_buf, len); } if (message == NULL) diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -3,12 +3,1445 @@ of int.__float__, etc., that take and return unicode objects */ #include "Python.h" -#include "../Objects/stringlib/unicodedefs.h" +#include <locale.h> +/* Raises an exception about an unknown presentation type for this + * type. */ -#define FORMAT_STRING _PyUnicode_FormatAdvanced -#define FORMAT_LONG _PyLong_FormatAdvanced -#define FORMAT_FLOAT _PyFloat_FormatAdvanced -#define FORMAT_COMPLEX _PyComplex_FormatAdvanced +static void +unknown_presentation_type(Py_UCS4 presentation_type, + const char* type_name) +{ + /* %c might be out-of-range, hence the two cases. */ + if (presentation_type > 32 && presentation_type < 128) + PyErr_Format(PyExc_ValueError, + "Unknown format code '%c' " + "for object of type '%.200s'", + (char)presentation_type, + type_name); + else + PyErr_Format(PyExc_ValueError, + "Unknown format code '\\x%x' " + "for object of type '%.200s'", + (unsigned int)presentation_type, + type_name); +} -#include "../Objects/stringlib/formatter.h" +static void +invalid_comma_type(Py_UCS4 presentation_type) +{ + if (presentation_type > 32 && presentation_type < 128) + PyErr_Format(PyExc_ValueError, + "Cannot specify ',' with '%c'.", + (char)presentation_type); + else + PyErr_Format(PyExc_ValueError, + "Cannot specify ',' with '\\x%x'.", + (unsigned int)presentation_type); +} + +/* + get_integer consumes 0 or more decimal digit characters from an + input string, updates *result with the corresponding positive + integer, and returns the number of digits consumed. + + returns -1 on error. +*/ +static int +get_integer(PyObject *str, Py_ssize_t *pos, Py_ssize_t end, + Py_ssize_t *result) +{ + Py_ssize_t accumulator, digitval, oldaccumulator; + int numdigits; + accumulator = numdigits = 0; + for (;;(*pos)++, numdigits++) { + if (*pos >= end) + break; + digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str, *pos)); + if (digitval < 0) + break; + /* + This trick was copied from old Unicode format code. It's cute, + but would really suck on an old machine with a slow divide + implementation. Fortunately, in the normal case we do not + expect too many digits. + */ + oldaccumulator = accumulator; + accumulator *= 10; + if ((accumulator+10)/10 != oldaccumulator+1) { + PyErr_Format(PyExc_ValueError, + "Too many decimal digits in format string"); + return -1; + } + accumulator += digitval; + } + *result = accumulator; + return numdigits; +} + +/************************************************************************/ +/*********** standard format specifier parsing **************************/ +/************************************************************************/ + +/* returns true if this character is a specifier alignment token */ +Py_LOCAL_INLINE(int) +is_alignment_token(Py_UCS4 c) +{ + switch (c) { + case '<': case '>': case '=': case '^': + return 1; + default: + return 0; + } +} + +/* returns true if this character is a sign element */ +Py_LOCAL_INLINE(int) +is_sign_element(Py_UCS4 c) +{ + switch (c) { + case ' ': case '+': case '-': + return 1; + default: + return 0; + } +} + + +typedef struct { + Py_UCS4 fill_char; + Py_UCS4 align; + int alternate; + Py_UCS4 sign; + Py_ssize_t width; + int thousands_separators; + Py_ssize_t precision; + Py_UCS4 type; +} InternalFormatSpec; + +#if 0 +/* Occassionally useful for debugging. Should normally be commented out. */ +static void +DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format) +{ + printf("internal format spec: fill_char %d\n", format->fill_char); + printf("internal format spec: align %d\n", format->align); + printf("internal format spec: alternate %d\n", format->alternate); + printf("internal format spec: sign %d\n", format->sign); + printf("internal format spec: width %zd\n", format->width); + printf("internal format spec: thousands_separators %d\n", + format->thousands_separators); + printf("internal format spec: precision %zd\n", format->precision); + printf("internal format spec: type %c\n", format->type); + printf("\n"); +} +#endif + + +/* + ptr points to the start of the format_spec, end points just past its end. + fills in format with the parsed information. + returns 1 on success, 0 on failure. + if failure, sets the exception +*/ +static int +parse_internal_render_format_spec(PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end, + InternalFormatSpec *format, + char default_type, + char default_align) +{ + Py_ssize_t pos = start; + /* end-pos is used throughout this code to specify the length of + the input string */ +#define READ_spec(index) PyUnicode_READ_CHAR(format_spec, index) + + Py_ssize_t consumed; + int align_specified = 0; + + format->fill_char = '\0'; + format->align = default_align; + format->alternate = 0; + format->sign = '\0'; + format->width = -1; + format->thousands_separators = 0; + format->precision = -1; + format->type = default_type; + + /* If the second char is an alignment token, + then parse the fill char */ + if (end-pos >= 2 && is_alignment_token(READ_spec(pos+1))) { + format->align = READ_spec(pos+1); + format->fill_char = READ_spec(pos); + align_specified = 1; + pos += 2; + } + else if (end-pos >= 1 && is_alignment_token(READ_spec(pos))) { + format->align = READ_spec(pos); + align_specified = 1; + ++pos; + } + + /* Parse the various sign options */ + if (end-pos >= 1 && is_sign_element(READ_spec(pos))) { + format->sign = READ_spec(pos); + ++pos; + } + + /* If the next character is #, we're in alternate mode. This only + applies to integers. */ + if (end-pos >= 1 && READ_spec(pos) == '#') { + format->alternate = 1; + ++pos; + } + + /* The special case for 0-padding (backwards compat) */ + if (format->fill_char == '\0' && end-pos >= 1 && READ_spec(pos) == '0') { + format->fill_char = '0'; + if (!align_specified) { + format->align = '='; + } + ++pos; + } + + consumed = get_integer(format_spec, &pos, end, &format->width); + if (consumed == -1) + /* Overflow error. Exception already set. */ + return 0; + + /* If consumed is 0, we didn't consume any characters for the + width. In that case, reset the width to -1, because + get_integer() will have set it to zero. -1 is how we record + that the width wasn't specified. */ + if (consumed == 0) + format->width = -1; + + /* Comma signifies add thousands separators */ + if (end-pos && READ_spec(pos) == ',') { + format->thousands_separators = 1; + ++pos; + } + + /* Parse field precision */ + if (end-pos && READ_spec(pos) == '.') { + ++pos; + + consumed = get_integer(format_spec, &pos, end, &format->precision); + if (consumed == -1) + /* Overflow error. Exception already set. */ + return 0; + + /* Not having a precision after a dot is an error. */ + if (consumed == 0) { + PyErr_Format(PyExc_ValueError, + "Format specifier missing precision"); + return 0; + } + + } + + /* Finally, parse the type field. */ + + if (end-pos > 1) { + /* More than one char remain, invalid conversion spec. */ + PyErr_Format(PyExc_ValueError, "Invalid conversion specification"); + return 0; + } + + if (end-pos == 1) { + format->type = READ_spec(pos); + ++pos; + } + + /* Do as much validating as we can, just by looking at the format + specifier. Do not take into account what type of formatting + we're doing (int, float, string). */ + + if (format->thousands_separators) { + switch (format->type) { + case 'd': + case 'e': + case 'f': + case 'g': + case 'E': + case 'G': + case '%': + case 'F': + case '\0': + /* These are allowed. See PEP 378.*/ + break; + default: + invalid_comma_type(format->type); + return 0; + } + } + + if (format->fill_char > 127 || format->align > 127 || + format->sign > 127) { + PyErr_SetString(PyExc_ValueError, "fill character too large"); + return 0; + } + + return 1; +} + +/* Calculate the padding needed. */ +static void +calc_padding(Py_ssize_t nchars, Py_ssize_t width, Py_UCS4 align, + Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding, + Py_ssize_t *n_total) +{ + if (width >= 0) { + if (nchars > width) + *n_total = nchars; + else + *n_total = width; + } + else { + /* not specified, use all of the chars and no more */ + *n_total = nchars; + } + + /* Figure out how much leading space we need, based on the + aligning */ + if (align == '>') + *n_lpadding = *n_total - nchars; + else if (align == '^') + *n_lpadding = (*n_total - nchars) / 2; + else if (align == '<' || align == '=') + *n_lpadding = 0; + else { + /* We should never have an unspecified alignment. */ + *n_lpadding = 0; + assert(0); + } + + *n_rpadding = *n_total - nchars - *n_lpadding; +} + +static void +unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch) +{ + int kind = PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + while (start < end) + PyUnicode_WRITE(kind, data, start++, ch); +} + +/* Do the padding, and return a pointer to where the caller-supplied + content goes. */ +static Py_ssize_t +fill_padding(PyObject *s, Py_ssize_t start, Py_ssize_t nchars, + Py_UCS4 fill_char, Py_ssize_t n_lpadding, + Py_ssize_t n_rpadding) +{ + /* Pad on left. */ + if (n_lpadding) + unicode_fill(s, start, start + n_lpadding, fill_char); + + /* Pad on right. */ + if (n_rpadding) + unicode_fill(s, start + nchars + n_lpadding, + start + nchars + n_lpadding + n_rpadding, fill_char); + + /* Pointer to the user content. */ + return start + n_lpadding; +} + +/************************************************************************/ +/*********** common routines for numeric formatting *********************/ +/************************************************************************/ + +/* Locale type codes. */ +#define LT_CURRENT_LOCALE 0 +#define LT_DEFAULT_LOCALE 1 +#define LT_NO_LOCALE 2 + +/* Locale info needed for formatting integers and the part of floats + before and including the decimal. Note that locales only support + 8-bit chars, not unicode. */ +typedef struct { + char *decimal_point; + char *thousands_sep; + char *grouping; +} LocaleInfo; + +/* describes the layout for an integer, see the comment in + calc_number_widths() for details */ +typedef struct { + Py_ssize_t n_lpadding; + Py_ssize_t n_prefix; + Py_ssize_t n_spadding; + Py_ssize_t n_rpadding; + char sign; + Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */ + Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including + any grouping chars. */ + Py_ssize_t n_decimal; /* 0 if only an integer */ + Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part, + excluding the decimal itself, if + present. */ + + /* These 2 are not the widths of fields, but are needed by + STRINGLIB_GROUPING. */ + Py_ssize_t n_digits; /* The number of digits before a decimal + or exponent. */ + Py_ssize_t n_min_width; /* The min_width we used when we computed + the n_grouped_digits width. */ +} NumberFieldWidths; + + +/* Given a number of the form: + digits[remainder] + where ptr points to the start and end points to the end, find where + the integer part ends. This could be a decimal, an exponent, both, + or neither. + If a decimal point is present, set *has_decimal and increment + remainder beyond it. + Results are undefined (but shouldn't crash) for improperly + formatted strings. +*/ +static void +parse_number(PyObject *s, Py_ssize_t pos, Py_ssize_t end, + Py_ssize_t *n_remainder, int *has_decimal) +{ + Py_ssize_t remainder; + + while (pos<end && isdigit(PyUnicode_READ_CHAR(s, pos))) + ++pos; + remainder = pos; + + /* Does remainder start with a decimal point? */ + *has_decimal = pos<end && PyUnicode_READ_CHAR(s, remainder) == '.'; + + /* Skip the decimal point. */ + if (*has_decimal) + remainder++; + + *n_remainder = end - remainder; +} + +/* not all fields of format are used. for example, precision is + unused. should this take discrete params in order to be more clear + about what it does? or is passing a single format parameter easier + and more efficient enough to justify a little obfuscation? */ +static Py_ssize_t +calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, + Py_UCS4 sign_char, PyObject *number, Py_ssize_t n_start, + Py_ssize_t n_end, Py_ssize_t n_remainder, + int has_decimal, const LocaleInfo *locale, + const InternalFormatSpec *format) +{ + Py_ssize_t n_non_digit_non_padding; + Py_ssize_t n_padding; + + spec->n_digits = n_end - n_start - n_remainder - (has_decimal?1:0); + spec->n_lpadding = 0; + spec->n_prefix = n_prefix; + spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0; + spec->n_remainder = n_remainder; + spec->n_spadding = 0; + spec->n_rpadding = 0; + spec->sign = '\0'; + spec->n_sign = 0; + + /* the output will look like: + | | + | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> | + | | + + sign is computed from format->sign and the actual + sign of the number + + prefix is given (it's for the '0x' prefix) + + digits is already known + + the total width is either given, or computed from the + actual digits + + only one of lpadding, spadding, and rpadding can be non-zero, + and it's calculated from the width and other fields + */ + + /* compute the various parts we're going to write */ + switch (format->sign) { + case '+': + /* always put a + or - */ + spec->n_sign = 1; + spec->sign = (sign_char == '-' ? '-' : '+'); + break; + case ' ': + spec->n_sign = 1; + spec->sign = (sign_char == '-' ? '-' : ' '); + break; + default: + /* Not specified, or the default (-) */ + if (sign_char == '-') { + spec->n_sign = 1; + spec->sign = '-'; + } + } + + /* The number of chars used for non-digits and non-padding. */ + n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal + + spec->n_remainder; + + /* min_width can go negative, that's okay. format->width == -1 means + we don't care. */ + if (format->fill_char == '0' && format->align == '=') + spec->n_min_width = format->width - n_non_digit_non_padding; + else + spec->n_min_width = 0; + + if (spec->n_digits == 0) + /* This case only occurs when using 'c' formatting, we need + to special case it because the grouping code always wants + to have at least one character. */ + spec->n_grouped_digits = 0; + else + spec->n_grouped_digits = _PyUnicode_InsertThousandsGrouping( + PyUnicode_1BYTE_KIND, NULL, 0, NULL, + spec->n_digits, spec->n_min_width, + locale->grouping, locale->thousands_sep); + + /* Given the desired width and the total of digit and non-digit + space we consume, see if we need any padding. format->width can + be negative (meaning no padding), but this code still works in + that case. */ + n_padding = format->width - + (n_non_digit_non_padding + spec->n_grouped_digits); + if (n_padding > 0) { + /* Some padding is needed. Determine if it's left, space, or right. */ + switch (format->align) { + case '<': + spec->n_rpadding = n_padding; + break; + case '^': + spec->n_lpadding = n_padding / 2; + spec->n_rpadding = n_padding - spec->n_lpadding; + break; + case '=': + spec->n_spadding = n_padding; + break; + case '>': + spec->n_lpadding = n_padding; + break; + default: + /* Shouldn't get here, but treat it as '>' */ + spec->n_lpadding = n_padding; + assert(0); + break; + } + } + return spec->n_lpadding + spec->n_sign + spec->n_prefix + + spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + + spec->n_remainder + spec->n_rpadding; +} + +/* Fill in the digit parts of a numbers's string representation, + as determined in calc_number_widths(). + No error checking, since we know the buffer is the correct size. */ +static void +fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec, + PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end, + PyObject *prefix, Py_ssize_t p_start, Py_UCS4 fill_char, + LocaleInfo *locale, int toupper) +{ + /* Used to keep track of digits, decimal, and remainder. */ + Py_ssize_t d_pos = d_start; + unsigned int kind = PyUnicode_KIND(out); + void *data = PyUnicode_DATA(out); + +#ifndef NDEBUG + Py_ssize_t r; +#endif + + if (spec->n_lpadding) { + unicode_fill(out, pos, pos + spec->n_lpadding, fill_char); + pos += spec->n_lpadding; + } + if (spec->n_sign == 1) { + PyUnicode_WRITE(kind, data, pos++, spec->sign); + } + if (spec->n_prefix) { + PyUnicode_CopyCharacters(out, pos, prefix, p_start, spec->n_prefix); + if (toupper) { + Py_ssize_t t; + /* XXX if the upper-case prefix is wider than the target + buffer, the caller should have allocated a wider string, + but currently doesn't. */ + for (t = 0; t < spec->n_prefix; ++t) + PyUnicode_WRITE(kind, data, pos + t, + Py_UNICODE_TOUPPER( + PyUnicode_READ(kind, data, pos + t))); + } + pos += spec->n_prefix; + } + if (spec->n_spadding) { + unicode_fill(out, pos, pos + spec->n_spadding, fill_char); + pos += spec->n_spadding; + } + + /* Only for type 'c' special case, it has no digits. */ + if (spec->n_digits != 0) { + /* Fill the digits with InsertThousandsGrouping. */ + char *pdigits = PyUnicode_DATA(digits); + if (PyUnicode_KIND(digits) < kind) { + pdigits = _PyUnicode_AsKind(digits, kind); + if (pdigits == NULL) { + /* XXX report exception */ + Py_FatalError("out of memory"); + return; + } + } +#ifndef NDEBUG + r = +#endif + _PyUnicode_InsertThousandsGrouping( + kind, + (char*)data + PyUnicode_KIND_SIZE(kind, pos), + spec->n_grouped_digits, + pdigits + PyUnicode_KIND_SIZE(kind, d_pos), + spec->n_digits, spec->n_min_width, + locale->grouping, locale->thousands_sep); +#ifndef NDEBUG + assert(r == spec->n_grouped_digits); +#endif + if (PyUnicode_KIND(digits) < kind) + PyMem_Free(pdigits); + d_pos += spec->n_digits; + } + if (toupper) { + Py_ssize_t t; + for (t = 0; t < spec->n_grouped_digits; ++t) + PyUnicode_WRITE(kind, data, pos + t, + Py_UNICODE_TOUPPER( + PyUnicode_READ(kind, data, pos + t))); + } + pos += spec->n_grouped_digits; + + if (spec->n_decimal) { + Py_ssize_t t; + for (t = 0; t < spec->n_decimal; ++t) + PyUnicode_WRITE(kind, data, pos + t, + locale->decimal_point[t]); + pos += spec->n_decimal; + d_pos += 1; + } + + if (spec->n_remainder) { + PyUnicode_CopyCharacters(out, pos, digits, d_pos, spec->n_remainder); + pos += spec->n_remainder; + d_pos += spec->n_remainder; + } + + if (spec->n_rpadding) { + unicode_fill(out, pos, pos + spec->n_rpadding, fill_char); + pos += spec->n_rpadding; + } +} + +static char no_grouping[1] = {CHAR_MAX}; + +/* Find the decimal point character(s?), thousands_separator(s?), and + grouping description, either for the current locale if type is + LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or + none if LT_NO_LOCALE. */ +static void +get_locale_info(int type, LocaleInfo *locale_info) +{ + switch (type) { + case LT_CURRENT_LOCALE: { + struct lconv *locale_data = localeconv(); + locale_info->decimal_point = locale_data->decimal_point; + locale_info->thousands_sep = locale_data->thousands_sep; + locale_info->grouping = locale_data->grouping; + break; + } + case LT_DEFAULT_LOCALE: + locale_info->decimal_point = "."; + locale_info->thousands_sep = ","; + locale_info->grouping = "\3"; /* Group every 3 characters. The + (implicit) trailing 0 means repeat + infinitely. */ + break; + case LT_NO_LOCALE: + locale_info->decimal_point = "."; + locale_info->thousands_sep = ""; + locale_info->grouping = no_grouping; + break; + default: + assert(0); + } +} + +/************************************************************************/ +/*********** string formatting ******************************************/ +/************************************************************************/ + +static PyObject * +format_string_internal(PyObject *value, const InternalFormatSpec *format) +{ + Py_ssize_t lpad; + Py_ssize_t rpad; + Py_ssize_t total; + Py_ssize_t pos; + Py_ssize_t len = PyUnicode_GET_SIZE(value); + PyObject *result = NULL; + int maxchar = 127; + + /* sign is not allowed on strings */ + if (format->sign != '\0') { + PyErr_SetString(PyExc_ValueError, + "Sign not allowed in string format specifier"); + goto done; + } + + /* alternate is not allowed on strings */ + if (format->alternate) { + PyErr_SetString(PyExc_ValueError, + "Alternate form (#) not allowed in string format " + "specifier"); + goto done; + } + + /* '=' alignment not allowed on strings */ + if (format->align == '=') { + PyErr_SetString(PyExc_ValueError, + "'=' alignment not allowed " + "in string format specifier"); + goto done; + } + + /* if precision is specified, output no more that format.precision + characters */ + if (format->precision >= 0 && len >= format->precision) { + len = format->precision; + } + + calc_padding(len, format->width, format->align, &lpad, &rpad, &total); + + /* allocate the resulting string */ + result = PyUnicode_New(total, maxchar); + if (result == NULL) + goto done; + + /* Write into that space. First the padding. */ + pos = fill_padding(result, 0, len, + format->fill_char=='\0'?' ':format->fill_char, + lpad, rpad); + + /* Then the source string. */ + PyUnicode_CopyCharacters(result, pos, value, 0, len); + +done: + return result; +} + + +/************************************************************************/ +/*********** long formatting ********************************************/ +/************************************************************************/ + +typedef PyObject* +(*IntOrLongToString)(PyObject *value, int base); + +static PyObject * +format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, + IntOrLongToString tostring) +{ + PyObject *result = NULL; + int maxchar = 127; + PyObject *tmp = NULL; + Py_ssize_t inumeric_chars; + Py_UCS4 sign_char = '\0'; + Py_ssize_t n_digits; /* count of digits need from the computed + string */ + Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which + produces non-digits */ + Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */ + Py_ssize_t n_total; + Py_ssize_t prefix; + NumberFieldWidths spec; + long x; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + /* no precision allowed on integers */ + if (format->precision != -1) { + PyErr_SetString(PyExc_ValueError, + "Precision not allowed in integer format specifier"); + goto done; + } + + /* special case for character formatting */ + if (format->type == 'c') { + /* error to specify a sign */ + if (format->sign != '\0') { + PyErr_SetString(PyExc_ValueError, + "Sign not allowed with integer" + " format specifier 'c'"); + goto done; + } + + /* taken from unicodeobject.c formatchar() */ + /* Integer input truncated to a character */ +/* XXX: won't work for int */ + x = PyLong_AsLong(value); + if (x == -1 && PyErr_Occurred()) + goto done; + if (x < 0 || x > 0x10ffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000) " + "(wide Python build)"); + goto done; + } + tmp = PyUnicode_FromOrdinal(x); + inumeric_chars = 0; + n_digits = 1; + if (x > maxchar) + maxchar = x; + + /* As a sort-of hack, we tell calc_number_widths that we only + have "remainder" characters. calc_number_widths thinks + these are characters that don't get formatted, only copied + into the output string. We do this for 'c' formatting, + because the characters are likely to be non-digits. */ + n_remainder = 1; + } + else { + int base; + int leading_chars_to_skip = 0; /* Number of characters added by + PyNumber_ToBase that we want to + skip over. */ + + /* Compute the base and how many characters will be added by + PyNumber_ToBase */ + switch (format->type) { + case 'b': + base = 2; + leading_chars_to_skip = 2; /* 0b */ + break; + case 'o': + base = 8; + leading_chars_to_skip = 2; /* 0o */ + break; + case 'x': + case 'X': + base = 16; + leading_chars_to_skip = 2; /* 0x */ + break; + default: /* shouldn't be needed, but stops a compiler warning */ + case 'd': + case 'n': + base = 10; + break; + } + + /* The number of prefix chars is the same as the leading + chars to skip */ + if (format->alternate) + n_prefix = leading_chars_to_skip; + + /* Do the hard part, converting to a string in a given base */ + tmp = tostring(value, base); + if (tmp == NULL || PyUnicode_READY(tmp) == -1) + goto done; + + inumeric_chars = 0; + n_digits = PyUnicode_GET_LENGTH(tmp); + + prefix = inumeric_chars; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(tmp, inumeric_chars) == '-') { + sign_char = '-'; + ++prefix; + ++leading_chars_to_skip; + } + + /* Skip over the leading chars (0x, 0b, etc.) */ + n_digits -= leading_chars_to_skip; + inumeric_chars += leading_chars_to_skip; + } + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Calculate how much memory we'll need. */ + n_total = calc_number_widths(&spec, n_prefix, sign_char, tmp, inumeric_chars, + inumeric_chars + n_digits, n_remainder, 0, &locale, format); + + /* Allocate the memory. */ + result = PyUnicode_New(n_total, maxchar); + if (!result) + goto done; + + /* Populate the memory. */ + fill_number(result, 0, &spec, tmp, inumeric_chars, inumeric_chars + n_digits, + tmp, prefix, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, format->type == 'X'); + +done: + Py_XDECREF(tmp); + return result; +} + +/************************************************************************/ +/*********** float formatting *******************************************/ +/************************************************************************/ + +static PyObject* +strtounicode(char *charbuffer, Py_ssize_t len) +{ + return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, charbuffer, len); +} + +/* much of this is taken from unicodeobject.c */ +static PyObject * +format_float_internal(PyObject *value, + const InternalFormatSpec *format) +{ + char *buf = NULL; /* buffer returned from PyOS_double_to_string */ + Py_ssize_t n_digits; + Py_ssize_t n_remainder; + Py_ssize_t n_total; + int has_decimal; + double val; + Py_ssize_t precision = format->precision; + Py_ssize_t default_precision = 6; + Py_UCS4 type = format->type; + int add_pct = 0; + Py_ssize_t index; + NumberFieldWidths spec; + int flags = 0; + PyObject *result = NULL; + int maxchar = 127; + Py_UCS4 sign_char = '\0'; + int float_type; /* Used to see if we have a nan, inf, or regular float. */ + PyObject *unicode_tmp = NULL; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + if (format->alternate) + flags |= Py_DTSF_ALT; + + if (type == '\0') { + /* Omitted type specifier. Behaves in the same way as repr(x) + and str(x) if no precision is given, else like 'g', but with + at least one digit after the decimal point. */ + flags |= Py_DTSF_ADD_DOT_0; + type = 'r'; + default_precision = 0; + } + + if (type == 'n') + /* 'n' is the same as 'g', except for the locale used to + format the result. We take care of that later. */ + type = 'g'; + + val = PyFloat_AsDouble(value); + if (val == -1.0 && PyErr_Occurred()) + goto done; + + if (type == '%') { + type = 'f'; + val *= 100; + add_pct = 1; + } + + if (precision < 0) + precision = default_precision; + else if (type == 'r') + type = 'g'; + + /* Cast "type", because if we're in unicode we need to pass a + 8-bit char. This is safe, because we've restricted what "type" + can be. */ + buf = PyOS_double_to_string(val, (char)type, precision, flags, + &float_type); + if (buf == NULL) + goto done; + n_digits = strlen(buf); + + if (add_pct) { + /* We know that buf has a trailing zero (since we just called + strlen() on it), and we don't use that fact any more. So we + can just write over the trailing zero. */ + buf[n_digits] = '%'; + n_digits += 1; + } + + /* Since there is no unicode version of PyOS_double_to_string, + just use the 8 bit version and then convert to unicode. */ + unicode_tmp = strtounicode(buf, n_digits); + if (unicode_tmp == NULL) + goto done; + index = 0; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(unicode_tmp, index) == '-') { + sign_char = '-'; + ++index; + --n_digits; + } + + /* Determine if we have any "remainder" (after the digits, might include + decimal or exponent or both (or neither)) */ + parse_number(unicode_tmp, index, index + n_digits, &n_remainder, &has_decimal); + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Calculate how much memory we'll need. */ + n_total = calc_number_widths(&spec, 0, sign_char, unicode_tmp, index, + index + n_digits, n_remainder, has_decimal, + &locale, format); + + /* Allocate the memory. */ + result = PyUnicode_New(n_total, maxchar); + if (result == NULL) + goto done; + + /* Populate the memory. */ + fill_number(result, 0, &spec, unicode_tmp, index, index + n_digits, + NULL, 0, + format->fill_char == '\0' ? ' ' : format->fill_char, &locale, + 0); + +done: + PyMem_Free(buf); + Py_DECREF(unicode_tmp); + return result; +} + +/************************************************************************/ +/*********** complex formatting *****************************************/ +/************************************************************************/ + +static PyObject * +format_complex_internal(PyObject *value, + const InternalFormatSpec *format) +{ + double re; + double im; + char *re_buf = NULL; /* buffer returned from PyOS_double_to_string */ + char *im_buf = NULL; /* buffer returned from PyOS_double_to_string */ + + InternalFormatSpec tmp_format = *format; + Py_ssize_t n_re_digits; + Py_ssize_t n_im_digits; + Py_ssize_t n_re_remainder; + Py_ssize_t n_im_remainder; + Py_ssize_t n_re_total; + Py_ssize_t n_im_total; + int re_has_decimal; + int im_has_decimal; + Py_ssize_t precision = format->precision; + Py_ssize_t default_precision = 6; + Py_UCS4 type = format->type; + Py_ssize_t i_re; + Py_ssize_t i_im; + NumberFieldWidths re_spec; + NumberFieldWidths im_spec; + int flags = 0; + PyObject *result = NULL; + int maxchar = 127; + int rkind; + void *rdata; + Py_ssize_t index; + Py_UCS4 re_sign_char = '\0'; + Py_UCS4 im_sign_char = '\0'; + int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ + int im_float_type; + int add_parens = 0; + int skip_re = 0; + Py_ssize_t lpad; + Py_ssize_t rpad; + Py_ssize_t total; + PyObject *re_unicode_tmp = NULL; + PyObject *im_unicode_tmp = NULL; + + /* Locale settings, either from the actual locale or + from a hard-code pseudo-locale */ + LocaleInfo locale; + + /* Zero padding is not allowed. */ + if (format->fill_char == '0') { + PyErr_SetString(PyExc_ValueError, + "Zero padding is not allowed in complex format " + "specifier"); + goto done; + } + + /* Neither is '=' alignment . */ + if (format->align == '=') { + PyErr_SetString(PyExc_ValueError, + "'=' alignment flag is not allowed in complex format " + "specifier"); + goto done; + } + + re = PyComplex_RealAsDouble(value); + if (re == -1.0 && PyErr_Occurred()) + goto done; + im = PyComplex_ImagAsDouble(value); + if (im == -1.0 && PyErr_Occurred()) + goto done; + + if (format->alternate) + flags |= Py_DTSF_ALT; + + if (type == '\0') { + /* Omitted type specifier. Should be like str(self). */ + type = 'r'; + default_precision = 0; + if (re == 0.0 && copysign(1.0, re) == 1.0) + skip_re = 1; + else + add_parens = 1; + } + + if (type == 'n') + /* 'n' is the same as 'g', except for the locale used to + format the result. We take care of that later. */ + type = 'g'; + + if (precision < 0) + precision = default_precision; + else if (type == 'r') + type = 'g'; + + /* Cast "type", because if we're in unicode we need to pass a + 8-bit char. This is safe, because we've restricted what "type" + can be. */ + re_buf = PyOS_double_to_string(re, (char)type, precision, flags, + &re_float_type); + if (re_buf == NULL) + goto done; + im_buf = PyOS_double_to_string(im, (char)type, precision, flags, + &im_float_type); + if (im_buf == NULL) + goto done; + + n_re_digits = strlen(re_buf); + n_im_digits = strlen(im_buf); + + /* Since there is no unicode version of PyOS_double_to_string, + just use the 8 bit version and then convert to unicode. */ + re_unicode_tmp = strtounicode(re_buf, n_re_digits); + if (re_unicode_tmp == NULL) + goto done; + i_re = 0; + + im_unicode_tmp = strtounicode(im_buf, n_im_digits); + if (im_unicode_tmp == NULL) + goto done; + i_im = 0; + + /* Is a sign character present in the output? If so, remember it + and skip it */ + if (PyUnicode_READ_CHAR(re_unicode_tmp, i_re) == '-') { + re_sign_char = '-'; + ++i_re; + --n_re_digits; + } + if (PyUnicode_READ_CHAR(im_unicode_tmp, i_im) == '-') { + im_sign_char = '-'; + ++i_im; + --n_im_digits; + } + + /* Determine if we have any "remainder" (after the digits, might include + decimal or exponent or both (or neither)) */ + parse_number(re_unicode_tmp, i_re, i_re + n_re_digits, + &n_re_remainder, &re_has_decimal); + parse_number(im_unicode_tmp, i_im, i_im + n_im_digits, + &n_im_remainder, &im_has_decimal); + + /* Determine the grouping, separator, and decimal point, if any. */ + get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : + (format->thousands_separators ? + LT_DEFAULT_LOCALE : + LT_NO_LOCALE), + &locale); + + /* Turn off any padding. We'll do it later after we've composed + the numbers without padding. */ + tmp_format.fill_char = '\0'; + tmp_format.align = '<'; + tmp_format.width = -1; + + /* Calculate how much memory we'll need. */ + n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, re_unicode_tmp, + i_re, i_re + n_re_digits, n_re_remainder, + re_has_decimal, &locale, &tmp_format); + + /* Same formatting, but always include a sign, unless the real part is + * going to be omitted, in which case we use whatever sign convention was + * requested by the original format. */ + if (!skip_re) + tmp_format.sign = '+'; + n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, im_unicode_tmp, + i_im, i_im + n_im_digits, n_im_remainder, + im_has_decimal, &locale, &tmp_format); + + if (skip_re) + n_re_total = 0; + + /* Add 1 for the 'j', and optionally 2 for parens. */ + calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, + format->width, format->align, &lpad, &rpad, &total); + + result = PyUnicode_New(total, maxchar); + if (result == NULL) + goto done; + rkind = PyUnicode_KIND(result); + rdata = PyUnicode_DATA(result); + + /* Populate the memory. First, the padding. */ + index = fill_padding(result, 0, + n_re_total + n_im_total + 1 + add_parens * 2, + format->fill_char=='\0' ? ' ' : format->fill_char, + lpad, rpad); + + if (add_parens) + PyUnicode_WRITE(rkind, rdata, index++, '('); + + if (!skip_re) { + fill_number(result, index, &re_spec, re_unicode_tmp, + i_re, i_re + n_re_digits, NULL, 0, 0, &locale, 0); + index += n_re_total; + } + fill_number(result, index, &im_spec, im_unicode_tmp, + i_im, i_im + n_im_digits, NULL, 0, 0, &locale, 0); + index += n_im_total; + PyUnicode_WRITE(rkind, rdata, index++, 'j'); + + if (add_parens) + PyUnicode_WRITE(rkind, rdata, index++, ')'); + +done: + PyMem_Free(re_buf); + PyMem_Free(im_buf); + Py_XDECREF(re_unicode_tmp); + Py_XDECREF(im_unicode_tmp); + return result; +} + +/************************************************************************/ +/*********** built in formatters ****************************************/ +/************************************************************************/ +PyObject * +_PyUnicode_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + InternalFormatSpec format; + PyObject *result = NULL; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, 's', '<')) + goto done; + + /* type conversion? */ + switch (format.type) { + case 's': + /* no type conversion needed, already a string. do the formatting */ + result = format_string_internal(obj, &format); + break; + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + return result; +} + +static PyObject* +format_int_or_long(PyObject* obj, PyObject* format_spec, + Py_ssize_t start, Py_ssize_t end, + IntOrLongToString tostring) +{ + PyObject *result = NULL; + PyObject *tmp = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, 'd', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case 'b': + case 'c': + case 'd': + case 'o': + case 'x': + case 'X': + case 'n': + /* no type conversion needed, already an int (or long). do + the formatting */ + result = format_int_or_long_internal(obj, &format, tostring); + break; + + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case '%': + /* convert to float */ + tmp = PyNumber_Float(obj); + if (tmp == NULL) + goto done; + result = format_float_internal(tmp, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + Py_XDECREF(tmp); + return result; +} + +/* Need to define long_format as a function that will convert a long + to a string. In 3.0, _PyLong_Format has the correct signature. */ +#define long_format _PyLong_Format + +PyObject * +_PyLong_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + return format_int_or_long(obj, format_spec, start, end, + long_format); +} + +PyObject * +_PyFloat_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + PyObject *result = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, '\0', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case '\0': /* No format code: like 'g', but with at least one decimal. */ + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case 'n': + case '%': + /* no conversion, already a float. do the formatting */ + result = format_float_internal(obj, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + return result; +} + +PyObject * +_PyComplex_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) +{ + PyObject *result = NULL; + InternalFormatSpec format; + + /* check for the special case of zero length format spec, make + it equivalent to str(obj) */ + if (start == end) { + result = PyObject_Str(obj); + goto done; + } + + /* parse the format_spec */ + if (!parse_internal_render_format_spec(format_spec, start, end, + &format, '\0', '>')) + goto done; + + /* type conversion? */ + switch (format.type) { + case '\0': /* No format code: like 'g', but with at least one decimal. */ + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + case 'n': + /* no conversion, already a complex. do the formatting */ + result = format_complex_internal(obj, &format); + break; + + default: + /* unknown */ + unknown_presentation_type(format.type, obj->ob_type->tp_name); + goto done; + } + +done: + return result; +} diff --git a/Python/getargs.c b/Python/getargs.c --- a/Python/getargs.c +++ b/Python/getargs.c @@ -546,9 +546,6 @@ -#define UNICODE_DEFAULT_ENCODING(arg) \ - _PyUnicode_AsDefaultEncodedString(arg) - /* Format an error message generated by convertsimple(). */ static char * @@ -611,7 +608,7 @@ const char *format = *p_format; char c = *format++; - PyObject *uarg; + char *sarg; switch (c) { @@ -838,8 +835,11 @@ case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); if (PyUnicode_Check(arg) && - PyUnicode_GET_SIZE(arg) == 1) - *p = PyUnicode_AS_UNICODE(arg)[0]; + PyUnicode_GET_LENGTH(arg) == 1) { + int kind = PyUnicode_KIND(arg); + void *data = PyUnicode_DATA(arg); + *p = PyUnicode_READ(kind, data, 0); + } else return converterr("a unicode character", arg, msgbuf, bufsize); break; @@ -889,13 +889,12 @@ if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + Py_ssize_t len; + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - PyBuffer_FillInfo(p, arg, - PyBytes_AS_STRING(uarg), PyBytes_GET_SIZE(uarg), - 1, 0); + PyBuffer_FillInfo(p, arg, sarg, len, 1, 0); } else { /* any buffer-like object */ char *buf; @@ -918,12 +917,13 @@ STORE_SIZE(0); } else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + Py_ssize_t len; + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyBytes_AS_STRING(uarg); - STORE_SIZE(PyBytes_GET_SIZE(uarg)); + *p = sarg; + STORE_SIZE(len); } else { /* any buffer-like object */ /* XXX Really? */ @@ -937,22 +937,22 @@ } else { /* "s" or "z" */ char **p = va_arg(*p_va, char **); - uarg = NULL; + Py_ssize_t len; + sarg = NULL; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { - uarg = UNICODE_DEFAULT_ENCODING(arg); - if (uarg == NULL) + sarg = PyUnicode_AsUTF8AndSize(arg, &len); + if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); - *p = PyBytes_AS_STRING(uarg); + *p = sarg; } else return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); - if (*p != NULL && uarg != NULL && - (Py_ssize_t) strlen(*p) != PyBytes_GET_SIZE(uarg)) + if (*p != NULL && sarg != NULL && (Py_ssize_t) strlen(*p) != len) return converterr( c == 'z' ? "str without null bytes or None" : "str without null bytes", @@ -976,6 +976,8 @@ } else if (PyUnicode_Check(arg)) { *p = PyUnicode_AS_UNICODE(arg); + if (*p == NULL) + RETURN_ERR_OCCURRED; STORE_SIZE(PyUnicode_GET_SIZE(arg)); } else @@ -987,6 +989,8 @@ *p = NULL; else if (PyUnicode_Check(arg)) { *p = PyUnicode_AS_UNICODE(arg); + if (*p == NULL) + RETURN_ERR_OCCURRED; if (Py_UNICODE_strlen(*p) != PyUnicode_GET_SIZE(arg)) return converterr( "str without null character or None", diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -118,12 +118,12 @@ #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UNICODE CACHEDIR_UNICODE[] = { +static const Py_UCS4 CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UNICODE PYC_TAG_UNICODE[] = { +static const Py_UCS4 PYC_TAG_UNICODE[] = { 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY @@ -762,7 +762,7 @@ static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); +static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -886,10 +886,10 @@ /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ -static Py_UNICODE* -rightmost_sep(Py_UNICODE *s) +static Py_UCS4* +rightmost_sep(Py_UCS4 *s) { - Py_UNICODE *found, c; + Py_UCS4 *found, c; for (found = NULL; (c = *s); s++) { if (c == SEP #ifdef ALTSEP @@ -912,15 +912,21 @@ foo.py -> __pycache__/foo.<tag>.pyc */ static PyObject* -make_compiled_pathname(Py_UNICODE *pathname, int debug) +make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UNICODE buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 buf[MAXPATHLEN]; size_t buflen = (size_t)MAXPATHLEN; - size_t len = Py_UNICODE_strlen(pathname); + size_t len; size_t i, save; - Py_UNICODE *pos; + Py_UCS4 *pos; int sep = SEP; + pathname = PyUnicode_AsUCS4Copy(pathstr); + if (!pathname) + return NULL; + len = Py_UCS4_strlen(pathname); + /* Sanity check that the buffer has roughly enough space to hold what will eventually be the full path to the compiled file. The 5 extra bytes include the slash afer __pycache__, the two extra dots, the @@ -930,8 +936,10 @@ sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) + if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { + PyMem_Free(pathname); return NULL; + } /* Find the last path separator and copy everything from the start of the source string up to and including the separator. @@ -943,24 +951,28 @@ else { sep = *pos; i = pos - pathname + 1; - Py_UNICODE_strncpy(buf, pathname, i); + Py_UCS4_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); - i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; + Py_UCS4_strcat(buf, CACHEDIR_UNICODE); + i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf + i, '.'); + Py_UCS4_strcat(buf, pathname + save); + pos = Py_UCS4_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; - Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); + + /* pathname is not used from here on. */ + PyMem_Free(pathname); + + Py_UCS4_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,7 +1022,7 @@ #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - len = Py_UNICODE_strlen(buf); + len = Py_UCS4_strlen(buf); if (len + 5 > buflen) return NULL; buf[len] = '.'; len++; @@ -1018,7 +1030,7 @@ buf[len] = 'y'; len++; buf[len] = debug ? 'c' : 'o'; len++; assert(len <= buflen); - return PyUnicode_FromUnicode(buf, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); } @@ -1033,14 +1045,16 @@ static PyObject* make_source_pathname(PyObject *pathobj) { - Py_UNICODE buf[MAXPATHLEN]; - Py_UNICODE *pathname; - Py_UNICODE *left, *right, *dot0, *dot1, sep; + Py_UCS4 buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 *left, *right, *dot0, *dot1, sep; size_t i, j; - if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) return NULL; - pathname = PyUnicode_AS_UNICODE(pathobj); + pathname = PyUnicode_AsUCS4Copy(pathobj); + if (!pathname) + return NULL; /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style @@ -1057,31 +1071,35 @@ left = pathname; else left++; - if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || - Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - return NULL; + if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || + Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) + goto error; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) - return NULL; - if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) - return NULL; + if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) + goto error; + if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) + goto error; /* Too many dots? */ - if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) - return NULL; + if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) + goto error; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); - Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); + Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); buf[i+j] = 'p'; buf[i+j+1] = 'y'; - return PyUnicode_FromUnicode(buf, i+j+2); + PyMem_Free(pathname); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); + error: + PyMem_Free(pathname); + return NULL; } /* Given a pathname for a Python source file, its time of last @@ -1250,6 +1268,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, struct stat *srcstat) { + Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ @@ -1267,18 +1286,23 @@ PyObject *cpathbytes; #endif PyObject *dirname; - Py_UNICODE *dirsep; + Py_UCS4 *dirsep; int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); + if (!cpathname_ucs4) + return; + dirsep = rightmost_sep(cpathname_ucs4); if (dirsep == NULL) { if (Py_VerboseFlag) PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); return; } - dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), - dirsep - PyUnicode_AS_UNICODE(cpathname)); + dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + cpathname_ucs4, + dirsep - cpathname_ucs4); + PyMem_Free(cpathname_ucs4); if (dirname == NULL) { PyErr_Clear(); return; @@ -1461,9 +1485,7 @@ goto error; } #endif - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - !Py_OptimizeFlag); + cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) fpc = check_compiled_module(pathname, st.st_mtime, cpathname); @@ -1512,16 +1534,18 @@ get_sourcefile(PyObject *filename) { Py_ssize_t len; - Py_UNICODE *fileuni; + Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; - len = PyUnicode_GET_SIZE(filename); + len = PyUnicode_GET_LENGTH(filename); if (len == 0) Py_RETURN_NONE; /* don't match *.pyc or *.pyo? */ - fileuni = PyUnicode_AS_UNICODE(filename); + fileuni = PyUnicode_AsUCS4Copy(filename); + if (!fileuni) + return NULL; if (len < 5 || fileuni[len-4] != '.' || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') @@ -1535,7 +1559,7 @@ py = make_source_pathname(filename); if (py == NULL) { PyErr_Clear(); - py = PyUnicode_FromUnicode(fileuni, len - 1); + py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); } if (py == NULL) goto error; @@ -1548,6 +1572,7 @@ error: PyErr_Clear(); unchanged: + PyMem_Free(fileuni); Py_INCREF(filename); return filename; } @@ -1739,8 +1764,7 @@ PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UNICODE buf[MAXPATHLEN+1]; - Py_ssize_t buflen = MAXPATHLEN+1; + Py_UCS4 buf[MAXPATHLEN+1]; PyObject *path_unicode, *filename; Py_ssize_t len; struct stat statbuf; @@ -1759,15 +1783,15 @@ else return 0; - len = PyUnicode_GET_SIZE(path_unicode); - if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + len = PyUnicode_GET_LENGTH(path_unicode); + if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); - return 0; /* Too long */ + PyErr_Clear(); + return 0; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); Py_DECREF(path_unicode); - if (Py_UNICODE_strlen(buf) != len) + if (Py_UCS4_strlen(buf) != len) return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ @@ -1804,10 +1828,14 @@ #endif ) buf[len++] = SEP; - Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); - len += PyUnicode_GET_SIZE(name); - - filename = PyUnicode_FromUnicode(buf, len); + if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + PyErr_Clear(); + return 0; + } + len += PyUnicode_GET_LENGTH(name); + + filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (filename == NULL) return -1; @@ -1989,6 +2017,12 @@ if (p_loader != NULL) *p_loader = NULL; + if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; @@ -2704,7 +2738,7 @@ int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, @@ -2718,37 +2752,47 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UNICODE buf[MAXPATHLEN+1]; + Py_UCS4 buf[MAXPATHLEN+1]; Py_ssize_t buflen; Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *parent, *head, *next, *tail, *inputname, *outputname; PyObject *parent_name, *ensure_name; - const Py_UNICODE *nameunicode; - - nameunicode = PyUnicode_AS_UNICODE(name); - - if (Py_UNICODE_strchr(nameunicode, SEP) != NULL + Py_ssize_t sep, altsep; + + if (PyUnicode_READY(name)) + return NULL; + + sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (sep == -2) + return NULL; #ifdef ALTSEP - || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL + altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (altsep == -2) + return NULL; +#else + altsep = -1; #endif - ) { + if (sep != -1 || altsep != -1) + { PyErr_SetString(PyExc_ImportError, "Import by filename is not supported."); return NULL; } parent = get_parent(globals, &parent_name, level); - if (parent == NULL) + if (parent == NULL) { return NULL; - - buflen = PyUnicode_GET_SIZE(parent_name); - if (buflen+1 > bufsize) { + } + + if (PyUnicode_READY(parent_name)) + return NULL; + buflen = PyUnicode_GET_LENGTH(parent_name); + if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, @@ -2799,7 +2843,8 @@ Py_DECREF(head); - ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, Py_UCS4_strlen(buf)); if (ensure_name == NULL) { Py_DECREF(tail); return NULL; @@ -2859,8 +2904,6 @@ static PyObject * get_parent(PyObject *globals, PyObject **p_name, int level) { - Py_UNICODE name[MAXPATHLEN+1]; - const Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *nameobj; static PyObject *namestr = NULL; @@ -2897,7 +2940,7 @@ "__package__ set to non-string"); return NULL; } - if (PyUnicode_GET_SIZE(pkgname) == 0) { + if (PyUnicode_GET_LENGTH(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); @@ -2905,12 +2948,8 @@ } goto return_none; } - if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Package name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); + Py_INCREF(pkgname); + nameobj = pkgname; } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); @@ -2922,74 +2961,71 @@ /* __path__ is set, so modname is already the package name */ int error; - if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + Py_INCREF(modname); + nameobj = modname; } else { /* Normal module, so work out the package name if any */ - Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); - Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); Py_ssize_t len; - int error; - if (lastdot == NULL && level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); + len = PyUnicode_FindChar(modname, '.', + 0, PyUnicode_GET_LENGTH(modname), -1); + if (len == -2) return NULL; - } - if (lastdot == NULL) { - error = PyDict_SetItem(globals, pkgstr, Py_None); - if (error) { + if (len < 0) { + if (level > 0) { + PyErr_SetString(PyExc_ValueError, + "Attempted relative import in non-package"); + return NULL; + } + if (PyDict_SetItem(globals, pkgstr, Py_None)) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } goto return_none; } - len = lastdot - start; - if (len+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); + pkgname = PyUnicode_Substring(modname, 0, len); + if (pkgname == NULL) return NULL; - } - Py_UNICODE_strncpy(name, start, len); - name[len] = '\0'; - pkgname = PyUnicode_FromUnicode(name, len); - if (pkgname == NULL) { - return NULL; - } - error = PyDict_SetItem(globals, pkgstr, pkgname); - Py_DECREF(pkgname); - if (error) { + if (PyDict_SetItem(globals, pkgstr, pkgname)) { + Py_DECREF(pkgname); PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + nameobj = pkgname; } } - while (--level > 0) { - Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); - if (dot == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); + if (level > 1) { + Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); + PyObject *newname; + while (--level > 0) { + dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); + if (dot == -2) { + Py_DECREF(nameobj); + return NULL; + } + if (dot < 0) { + Py_DECREF(nameobj); + PyErr_SetString(PyExc_ValueError, + "Attempted relative import beyond " + "toplevel package"); + return NULL; + } + end = dot; + } + newname = PyUnicode_Substring(nameobj, 0, end); + Py_DECREF(nameobj); + if (newname == NULL) return NULL; - } - *dot = '\0'; + nameobj = newname; } - nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); - if (nameobj == NULL) - return NULL; - modules = PyImport_GetModuleDict(); parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { @@ -3021,7 +3057,7 @@ If this is violated... Who cares? */ return_none: - nameobj = PyUnicode_FromUnicode(NULL, 0); + nameobj = PyUnicode_New(0, 0); if (nameobj == NULL) return NULL; *p_name = nameobj; @@ -3032,28 +3068,28 @@ static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - const Py_UNICODE *dot; + Py_UCS4 *dot; Py_ssize_t len; - Py_UNICODE *p; + Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UNICODE *nameuni; + const Py_UCS4 *nameuni; *p_outputname = NULL; - if (PyUnicode_GET_SIZE(inputname) == 0) { + if (PyUnicode_GET_LENGTH(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AS_UNICODE(inputname); + nameuni = PyUnicode_AsUCS4Copy(inputname); if (nameuni == NULL) return NULL; - dot = Py_UNICODE_strchr(nameuni, '.'); + dot = Py_UCS4_strchr(nameuni, '.'); if (dot != NULL) { len = dot - nameuni; if (len == 0) { @@ -3063,7 +3099,7 @@ } } else - len = PyUnicode_GET_SIZE(inputname); + len = PyUnicode_GET_LENGTH(inputname); if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, @@ -3076,14 +3112,16 @@ *p++ = '.'; *p_buflen += 1; } - Py_UNICODE_strncpy(p, nameuni, len); + Py_UCS4_strncpy(p, nameuni, len); p[len] = '\0'; *p_buflen += len; - fullname = PyUnicode_FromUnicode(buf, *p_buflen); + fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (fullname == NULL) return NULL; - name = PyUnicode_FromUnicode(p, len); + name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + p, len); if (name == NULL) { Py_DECREF(fullname); return NULL; @@ -3096,7 +3134,8 @@ result = import_submodule(altmod, name, name); Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); return NULL; @@ -3107,7 +3146,7 @@ return NULL; } Py_DECREF(mark_name); - Py_UNICODE_strncpy(buf, nameuni, len); + Py_UCS4_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } @@ -3125,7 +3164,8 @@ } if (dot != NULL) { - *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); return NULL; @@ -3166,7 +3206,7 @@ Py_DECREF(item); return 0; } - if (PyUnicode_AS_UNICODE(item)[0] == '*') { + if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; Py_DECREF(item); /* See if the package defines __all__ */ @@ -3304,7 +3344,7 @@ PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; - Py_UNICODE *name, *subname; + Py_UCS4 *name = NULL, *subname; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3321,7 +3361,7 @@ return NULL; } nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL) + if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, @@ -3343,8 +3383,12 @@ return NULL; } - name = PyUnicode_AS_UNICODE(nameobj); - subname = Py_UNICODE_strrchr(name, '.'); + name = PyUnicode_AsUCS4Copy(nameobj); + if (!name) { + Py_DECREF(nameobj); + return NULL; + } + subname = Py_UCS4_strrchr(name, '.'); if (subname == NULL) { Py_INCREF(nameobj); subnameobj = nameobj; @@ -3353,7 +3397,8 @@ PyObject *parentname, *parent; Py_ssize_t len; len = subname - name; - parentname = PyUnicode_FromUnicode(name, len); + parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + name, len); if (parentname == NULL) { goto error; } @@ -3370,8 +3415,9 @@ if (path_list == NULL) PyErr_Clear(); subname++; - len = PyUnicode_GET_SIZE(nameobj) - (len + 1); - subnameobj = PyUnicode_FromUnicode(subname, len); + len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); + subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); } if (subnameobj == NULL) goto error; @@ -3403,6 +3449,7 @@ error: imp_modules_reloading_clear(); Py_DECREF(nameobj); + PyMem_Free(name); return newm; } @@ -3910,9 +3957,7 @@ return NULL; } - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - debug); + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); if (cpathname == NULL) { @@ -4105,7 +4150,7 @@ &pathobj)) return -1; - if (PyUnicode_GET_SIZE(pathobj) == 0) { + if (PyUnicode_GET_LENGTH(pathobj) == 0) { PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } diff --git a/Python/marshal.c b/Python/marshal.c --- a/Python/marshal.c +++ b/Python/marshal.c @@ -311,9 +311,7 @@ } else if (PyUnicode_CheckExact(v)) { PyObject *utf8; - utf8 = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(v), - PyUnicode_GET_SIZE(v), - "surrogatepass"); + utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass"); if (utf8 == NULL) { p->depth--; p->error = WFERR_UNMARSHALLABLE; diff --git a/Python/peephole.c b/Python/peephole.c --- a/Python/peephole.c +++ b/Python/peephole.c @@ -183,24 +183,6 @@ break; case BINARY_SUBSCR: newconst = PyObject_GetItem(v, w); - /* #5057: if v is unicode, there might be differences between - wide and narrow builds in cases like '\U00012345'[0]. - Wide builds will return a non-BMP char, whereas narrow builds - will return a surrogate. In both the cases skip the - optimization in order to produce compatible pycs. - */ - if (newconst != NULL && - PyUnicode_Check(v) && PyUnicode_Check(newconst)) { - Py_UNICODE ch = PyUnicode_AS_UNICODE(newconst)[0]; -#ifdef Py_UNICODE_WIDE - if (ch > 0xFFFF) { -#else - if (ch >= 0xD800 && ch <= 0xDFFF) { -#endif - Py_DECREF(newconst); - return 0; - } - } break; case BINARY_LSHIFT: newconst = PyNumber_Lshift(v, w); diff --git a/Python/symtable.c b/Python/symtable.c --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1525,10 +1525,10 @@ */ PyObject *store_name; PyObject *name = (a->asname == NULL) ? a->name : a->asname; - const Py_UNICODE *base = PyUnicode_AS_UNICODE(name); - Py_UNICODE *dot = Py_UNICODE_strchr(base, '.'); - if (dot) { - store_name = PyUnicode_FromUnicode(base, dot - base); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), 1); + if (dot != -1) { + store_name = PyUnicode_Substring(name, 0, dot); if (!store_name) return 0; } diff --git a/Python/traceback.c b/Python/traceback.c --- a/Python/traceback.c +++ b/Python/traceback.c @@ -229,8 +229,8 @@ PyObject *lineobj = NULL; PyObject *res; char buf[MAXPATHLEN+1]; - Py_UNICODE *u, *p; - Py_ssize_t len; + int kind; + void *data; /* open the file */ if (filename == NULL) @@ -285,13 +285,16 @@ } /* remove the indentation of the line */ - u = PyUnicode_AS_UNICODE(lineobj); - len = PyUnicode_GET_SIZE(lineobj); - for (p=u; *p == ' ' || *p == '\t' || *p == '\014'; p++) - len--; - if (u != p) { + kind = PyUnicode_KIND(lineobj); + data = PyUnicode_DATA(lineobj); + for (i=0; i < PyUnicode_GET_LENGTH(lineobj); i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch != ' ' && ch != '\t' && ch != '\014') + break; + } + if (i) { PyObject *truncated; - truncated = PyUnicode_FromUnicode(p, len); + truncated = PyUnicode_Substring(lineobj, i, PyUnicode_GET_LENGTH(lineobj)); if (truncated) { Py_DECREF(lineobj); lineobj = truncated; @@ -476,13 +479,26 @@ static void dump_ascii(int fd, PyObject *text) { + PyASCIIObject *ascii = (PyASCIIObject *)text; Py_ssize_t i, size; int truncated; - Py_UNICODE *u; - char c; + int kind; + void *data; + Py_UCS4 ch; - size = PyUnicode_GET_SIZE(text); - u = PyUnicode_AS_UNICODE(text); + size = ascii->length; + kind = ascii->state.kind; + if (ascii->state.compact) { + if (ascii->state.ascii) + data = ((PyASCIIObject*)text) + 1; + else + data = ((PyCompactUnicodeObject*)text) + 1; + } + else { + data = ((PyUnicodeObject *)text)->data.any; + if (data == NULL) + return; + } if (MAX_STRING_LENGTH < size) { size = MAX_STRING_LENGTH; @@ -491,27 +507,28 @@ else truncated = 0; - for (i=0; i < size; i++, u++) { - if (*u < 128) { - c = (char)*u; + for (i=0; i < size; i++) { + ch = PyUnicode_READ(kind, data, i); + if (ch < 128) { + char c = (char)ch; write(fd, &c, 1); } - else if (*u < 256) { + else if (ch < 256) { PUTS(fd, "\\x"); - dump_hexadecimal(2, *u, fd); + dump_hexadecimal(2, ch, fd); } else #ifdef Py_UNICODE_WIDE - if (*u < 65536) + if (ch < 65536) #endif { PUTS(fd, "\\u"); - dump_hexadecimal(4, *u, fd); + dump_hexadecimal(4, ch, fd); #ifdef Py_UNICODE_WIDE } else { PUTS(fd, "\\U"); - dump_hexadecimal(8, *u, fd); + dump_hexadecimal(8, ch, fd); #endif } } @@ -542,7 +559,7 @@ } /* PyFrame_GetLineNumber() was introduced in Python 2.7.0 and 3.2.0 */ - lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti); + lineno = PyCode_Addr2Line(code, frame->f_lasti); PUTS(fd, ", line "); dump_decimal(fd, lineno); PUTS(fd, " in "); diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -51,6 +51,8 @@ _type_void_ptr = gdb.lookup_type('void').pointer() # void* _type_size_t = gdb.lookup_type('size_t') +_is_pep393 = 'data' in [f.name for f in gdb.lookup_type('PyUnicodeObject').target().fields()] + SIZEOF_VOID_P = _type_void_ptr.sizeof @@ -1123,11 +1125,30 @@ # Py_ssize_t length; /* Length of raw Unicode data in buffer */ # Py_UNICODE *str; /* Raw Unicode buffer */ field_length = long(self.field('length')) - field_str = self.field('str') + if _is_pep393: + # Python 3.3 and newer + may_have_surrogates = False + field_state = long(self.field('state')) + repr_kind = (field_state & 0xC) >> 2 + if repr_kind == 0: + # string is not ready + may_have_surrogates = True + field_str = self.field('wstr') + field_length = self.field('wstr_length') + elif repr_kind == 1: + field_str = self.field('data')['latin1'] + elif repr_kind == 2: + field_str = self.field('data')['ucs2'] + elif repr_kind == 3: + field_str = self.field('data')['ucs4'] + else: + # Python 3.2 and earlier + field_str = self.field('str') + may_have_surrogates = self.char_width() == 2 # Gather a list of ints from the Py_UNICODE array; these are either - # UCS-2 or UCS-4 code points: - if self.char_width() > 2: + # UCS-1, UCS-2 or UCS-4 code points: + if not may_have_surrogates: Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)] else: # A more elaborate routine if sizeof(Py_UNICODE) is 2 in the diff --git a/configure b/configure --- a/configure +++ b/configure @@ -767,7 +767,6 @@ with_libm with_libc enable_big_digits -with_wide_unicode with_computed_gotos ' ac_precious_vars='build_alias @@ -778,7 +777,8 @@ LDFLAGS LIBS CPPFLAGS -CPP' +CPP +CPPFLAGS' # Initialize some variables set by options. @@ -1438,7 +1438,6 @@ --with-fpectl enable SIGFPE catching --with-libm=STRING math library --with-libc=STRING C library - --with-wide-unicode Use 4-byte Unicode characters (default is 2 bytes) --with(out)-computed-gotos Use computed gotos in evaluation loop (enabled by default on supported compilers) @@ -12370,65 +12369,19 @@ $as_echo "$ac_cv_wchar_t_signed" >&6; } fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking what type to use for str" >&5 -$as_echo_n "checking what type to use for str... " >&6; } - -# Check whether --with-wide-unicode was given. -if test "${with_wide_unicode+set}" = set; then : - withval=$with_wide_unicode; -if test "$withval" != no -then unicode_size="4" -else unicode_size="2" -fi - -else - -case "$have_ucs4_tcl" in - yes) unicode_size="4";; - *) unicode_size="2" ;; -esac - -fi - - - -case "$unicode_size" in - 4) - $as_echo "#define Py_UNICODE_SIZE 4" >>confdefs.h - - ABIFLAGS="${ABIFLAGS}u" - ;; - *) $as_echo "#define Py_UNICODE_SIZE 2" >>confdefs.h - ;; -esac - - - # wchar_t is only usable if it maps to an unsigned type -if test "$unicode_size" = "$ac_cv_sizeof_wchar_t" \ +if test "$ac_cv_sizeof_wchar_t" -ge 2 \ -a "$ac_cv_wchar_t_signed" = "no" then - PY_UNICODE_TYPE="wchar_t" + HAVE_USABLE_WCHAR_T="yes" $as_echo "#define HAVE_USABLE_WCHAR_T 1" >>confdefs.h - $as_echo "#define PY_UNICODE_TYPE wchar_t" >>confdefs.h - -elif test "$ac_cv_sizeof_short" = "$unicode_size" -then - PY_UNICODE_TYPE="unsigned short" - $as_echo "#define PY_UNICODE_TYPE unsigned short" >>confdefs.h - -elif test "$ac_cv_sizeof_long" = "$unicode_size" -then - PY_UNICODE_TYPE="unsigned long" - $as_echo "#define PY_UNICODE_TYPE unsigned long" >>confdefs.h - -else - PY_UNICODE_TYPE="no type found" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PY_UNICODE_TYPE" >&5 -$as_echo "$PY_UNICODE_TYPE" >&6; } +else + HAVE_USABLE_WCHAR_T="no usable wchar_t found" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAVE_USABLE_WCHAR_T" >&5 +$as_echo "$HAVE_USABLE_WCHAR_T" >&6; } # check for endianness { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether byte ordering is bigendian" >&5 diff --git a/configure.in b/configure.in --- a/configure.in +++ b/configure.in @@ -3615,57 +3615,19 @@ AC_MSG_RESULT($ac_cv_wchar_t_signed) fi -AC_MSG_CHECKING(what type to use for str) -AC_ARG_WITH(wide-unicode, - AS_HELP_STRING([--with-wide-unicode], [Use 4-byte Unicode characters (default is 2 bytes)]), -[ -if test "$withval" != no -then unicode_size="4" -else unicode_size="2" -fi -], -[ -case "$have_ucs4_tcl" in - yes) unicode_size="4";; - *) unicode_size="2" ;; -esac -]) - -AH_TEMPLATE(Py_UNICODE_SIZE, - [Define as the size of the unicode type.]) -case "$unicode_size" in - 4) - AC_DEFINE(Py_UNICODE_SIZE, 4) - ABIFLAGS="${ABIFLAGS}u" - ;; - *) AC_DEFINE(Py_UNICODE_SIZE, 2) ;; -esac - -AH_TEMPLATE(PY_UNICODE_TYPE, - [Define as the integral type used for Unicode representation.]) - # wchar_t is only usable if it maps to an unsigned type -if test "$unicode_size" = "$ac_cv_sizeof_wchar_t" \ +if test "$ac_cv_sizeof_wchar_t" -ge 2 \ -a "$ac_cv_wchar_t_signed" = "no" then - PY_UNICODE_TYPE="wchar_t" + HAVE_USABLE_WCHAR_T="yes" AC_DEFINE(HAVE_USABLE_WCHAR_T, 1, [Define if you have a useable wchar_t type defined in wchar.h; useable means wchar_t must be an unsigned type with at least 16 bits. (see Include/unicodeobject.h).]) - AC_DEFINE(PY_UNICODE_TYPE,wchar_t) -elif test "$ac_cv_sizeof_short" = "$unicode_size" -then - PY_UNICODE_TYPE="unsigned short" - AC_DEFINE(PY_UNICODE_TYPE,unsigned short) -elif test "$ac_cv_sizeof_long" = "$unicode_size" -then - PY_UNICODE_TYPE="unsigned long" - AC_DEFINE(PY_UNICODE_TYPE,unsigned long) else - PY_UNICODE_TYPE="no type found" + HAVE_USABLE_WCHAR_T="no usable wchar_t found" fi -AC_MSG_RESULT($PY_UNICODE_TYPE) +AC_MSG_RESULT($HAVE_USABLE_WCHAR_T) # check for endianness AC_C_BIGENDIAN diff --git a/pyconfig.h.in b/pyconfig.h.in --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1111,18 +1111,12 @@ /* Define to printf format modifier for Py_ssize_t */ #undef PY_FORMAT_SIZE_T -/* Define as the integral type used for Unicode representation. */ -#undef PY_UNICODE_TYPE - /* Define if you want to build an interpreter with many run-time checks. */ #undef Py_DEBUG /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED -/* Define as the size of the unicode type. */ -#undef Py_UNICODE_SIZE - /* assume C89 semantics that RETSIGTYPE is always void */ #undef RETSIGTYPE -- Repository URL: http://hg.python.org/cpython