Mailman 3 May 2012 - Python-checkins

cpython: Issue #14744: Fix compilation on Windows
by victor.stinner May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/6abab1a103a6 changeset: 77230:6abab1a103a6 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue May 29 18:51:10 2012 +0200 summary: Issue #14744: Fix compilation on Windows files: Objects/longobject.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1687,8 +1687,8 @@ WRITE_DIGITS(Py_UCS2); } else { + Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - Py_UCS4 *p; WRITE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS @@ -1845,8 +1845,8 @@ WRITE_DIGITS(Py_UCS2); } else { + Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - Py_UCS4 *p; WRITE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS -- Repository URL: http://hg.python.org/cpython

1 0

cpython: #10839: add new test file that was omitted from checkin
by r.david.murray May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/08857f4e9709 changeset: 77229:08857f4e9709 user: R David Murray <rdmurray(a)bitdance.com> date: Tue May 29 12:31:11 2012 -0400 summary: #10839: add new test file that was omitted from checkin files: Lib/test/test_email/test_message.py | 18 +++++++++++++++++ 1 files changed, 18 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_email/test_message.py @@ -0,0 +1,18 @@ +import unittest +from email import policy +from test.test_email import TestEmailBase + + +class Test(TestEmailBase): + + policy = policy.default + + def test_error_on_setitem_if_max_count_exceeded(self): + m = self._str_msg("") + m['To'] = 'abc@xyz' + with self.assertRaises(ValueError): + m['To'] = 'xyz@abc' + + +if __name__ == '__main__': + unittest.main() -- Repository URL: http://hg.python.org/cpython

1 0

cpython: #10839: raise an error on add of duplicate unique headers in new email policies
by r.david.murray May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/d7881a371c41 changeset: 77228:d7881a371c41 user: R David Murray <rdmurray(a)bitdance.com> date: Tue May 29 09:14:44 2012 -0400 summary: #10839: raise an error on add of duplicate unique headers in new email policies This feature was supposed to be part of the initial email6 checkin, but it got lost in my big refactoring. In this patch I'm not providing an easy way to turn off the errors, but they only happen when a header is added programmatically, and it is almost never the right thing to do to allow the duplicate to be added. An application that needs to add duplicates of unique headers can create a policy subclass to allow it. files: Doc/library/email.policy.rst | 25 ++++++++++++++++++++++++ Lib/email/_policybase.py | 19 ++++++++++++++++++ Lib/email/message.py | 10 +++++++++ Lib/email/policy.py | 8 +++++++ 4 files changed, 62 insertions(+), 0 deletions(-) diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -196,6 +196,25 @@ custom ``Message`` objects) should also provide such an attribute, otherwise defects in parsed messages will raise unexpected errors. + .. method:: header_max_count(name) + + Return the maximum allowed number of headers named *name*. + + Called when a header is added to a :class:`~email.message.Message` + object. If the returned value is not ``0`` or ``None``, and there are + already a number of headers with the name *name* equal to the value + returned, a :exc:`ValueError` is raised. + + Because the default behavior of ``Message.__setitem__`` is to append the + value to the list of headers, it is easy to create duplicate headers + without realizing it. This method allows certain headers to be limited + in the number of instances of that header that may be added to a + ``Message`` programmatically. (The limit is not observed by the parser, + which will faithfully produce as many headers as exist in the message + being parsed.) + + The default implementation returns ``None`` for all header names. + .. method:: header_source_parse(sourcelines) The email package calls this method with a list of strings, each string @@ -366,6 +385,12 @@ The class provides the following concrete implementations of the abstract methods of :class:`Policy`: + .. method:: header_max_count(name) + + Returns the value of the + :attr:`~email.headerregistry.BaseHeader.max_count` attribute of the + specialized class used to represent the header with the given name. + .. method:: header_source_parse(sourcelines) The implementation of this method is the same as that for the diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -194,6 +194,25 @@ """ obj.defects.append(defect) + def header_max_count(self, name): + """Return the maximum allowed number of headers named 'name'. + + Called when a header is added to a Message object. If the returned + value is not 0 or None, and there are already a number of headers with + the name 'name' equal to the value returned, a ValueError is raised. + + Because the default behavior of Message's __setitem__ is to append the + value to the list of headers, it is easy to create duplicate headers + without realizing it. This method allows certain headers to be limited + in the number of instances of that header that may be added to a + Message programmatically. (The limit is not observed by the parser, + which will faithfully produce as many headers as exist in the message + being parsed.) + + The default implementation returns None for all header names. + """ + return None + @abc.abstractmethod def header_source_parse(self, sourcelines): """Given a list of linesep terminated strings constituting the lines of diff --git a/Lib/email/message.py b/Lib/email/message.py --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -346,6 +346,16 @@ Note: this does not overwrite an existing header with the same field name. Use __delitem__() first to delete any existing headers. """ + max_count = self.policy.header_max_count(name) + if max_count: + lname = name.lower() + found = 0 + for k, v in self._headers: + if k.lower() == lname: + found += 1 + if found >= max_count: + raise ValueError("There may be at most {} {} headers " + "in a message".format(max_count, name)) self._headers.append(self.policy.header_store_parse(name, val)) def __delitem__(self, name): diff --git a/Lib/email/policy.py b/Lib/email/policy.py --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -69,6 +69,14 @@ object.__setattr__(self, 'header_factory', HeaderRegistry()) super().__init__(**kw) + def header_max_count(self, name): + """+ + The implementation for this class returns the max_count attribute from + the specialized header class that would be used to construct a header + of type 'name'. + """ + return self.header_factory[name].max_count + # The logic of the next three methods is chosen such that it is possible to # switch a Message object between a Compat32 policy and a policy derived # from this class and have the results stay consistent. This allows a -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Issue #14007: make TreeBuilder an actual type exposed from _elementtree, and
by eli.bendersky May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/717632ae7b3f changeset: 77226:717632ae7b3f parent: 77223:22b56b0b8619 user: Eli Bendersky <eliben(a)gmail.com> date: Tue May 29 15:45:16 2012 +0300 summary: Issue #14007: make TreeBuilder an actual type exposed from _elementtree, and subclassable. files: Lib/test/test_xml_etree.py | 14 ++ Modules/_elementtree.c | 163 ++++++++++++------------ 2 files changed, 97 insertions(+), 80 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1979,6 +1979,20 @@ parser.feed(self.sample1) self.assertIsNone(parser.close()) + def test_subclass(self): + class MyTreeBuilder(ET.TreeBuilder): + def foobar(self, x): + return x * 2 + + tb = MyTreeBuilder() + self.assertEqual(tb.foobar(10), 20) + + parser = ET.XMLParser(target=tb) + parser.feed(self.sample1) + + e = parser.close() + self.assertEqual(e.tag, 'html') + # XXX in _elementtree, the constructor of TreeBuilder expects no # arguments @unittest.expectedFailure diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1803,23 +1803,22 @@ typedef struct { PyObject_HEAD - PyObject* root; /* root node (first created node) */ - - ElementObject* this; /* current node */ - ElementObject* last; /* most recently created node */ - - PyObject* data; /* data collector (string or list), or NULL */ - - PyObject* stack; /* element stack */ - Py_ssize_t index; /* current stack size (0=empty) */ + PyObject *root; /* root node (first created node) */ + + ElementObject *this; /* current node */ + ElementObject *last; /* most recently created node */ + + PyObject *data; /* data collector (string or list), or NULL */ + + PyObject *stack; /* element stack */ + Py_ssize_t index; /* current stack size (0 means empty) */ /* element tracing */ - PyObject* events; /* list of events, or NULL if not collecting */ - PyObject* start_event_obj; /* event objects (NULL to ignore) */ - PyObject* end_event_obj; - PyObject* start_ns_event_obj; - PyObject* end_ns_event_obj; - + PyObject *events; /* list of events, or NULL if not collecting */ + PyObject *start_event_obj; /* event objects (NULL to ignore) */ + PyObject *end_event_obj; + PyObject *start_ns_event_obj; + PyObject *end_ns_event_obj; } TreeBuilderObject; static PyTypeObject TreeBuilder_Type; @@ -1829,48 +1828,42 @@ /* -------------------------------------------------------------------- */ /* constructor and destructor */ -LOCAL(PyObject*) -treebuilder_new(void) +static PyObject * +treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - TreeBuilderObject* self; - - self = PyObject_New(TreeBuilderObject, &TreeBuilder_Type); - if (self == NULL) - return NULL; - - self->root = NULL; - - Py_INCREF(Py_None); - self->this = (ElementObject*) Py_None; - - Py_INCREF(Py_None); - self->last = (ElementObject*) Py_None; - - self->data = NULL; - - self->stack = PyList_New(20); - self->index = 0; - - self->events = NULL; - self->start_event_obj = self->end_event_obj = NULL; - self->start_ns_event_obj = self->end_ns_event_obj = NULL; - - ALLOC(sizeof(TreeBuilderObject), "create treebuilder"); - - return (PyObject*) self; + TreeBuilderObject *t = (TreeBuilderObject *)type->tp_alloc(type, 0); + if (t != NULL) { + t->root = NULL; + + Py_INCREF(Py_None); + t->this = (ElementObject *)Py_None; + Py_INCREF(Py_None); + t->last = (ElementObject *)Py_None; + + t->data = NULL; + t->stack = PyList_New(20); + if (!t->stack) { + Py_DECREF(t->this); + Py_DECREF(t->last); + return NULL; + } + t->index = 0; + + t->events = NULL; + t->start_event_obj = t->end_event_obj = NULL; + t->start_ns_event_obj = t->end_ns_event_obj = NULL; + } + return (PyObject *)t; } -static PyObject* -treebuilder(PyObject* self_, PyObject* args) +static int +treebuilder_init(PyObject *self, PyObject *args, PyObject *kwds) { - if (!PyArg_ParseTuple(args, ":TreeBuilder")) - return NULL; - - return treebuilder_new(); + return 0; } static void -treebuilder_dealloc(TreeBuilderObject* self) +treebuilder_dealloc(TreeBuilderObject *self) { Py_XDECREF(self->end_ns_event_obj); Py_XDECREF(self->start_ns_event_obj); @@ -1883,9 +1876,7 @@ Py_DECREF(self->this); Py_XDECREF(self->root); - RELEASE(sizeof(TreeBuilderObject), "destroy treebuilder"); - - PyObject_Del(self); + Py_TYPE(self)->tp_free((PyObject *)self); } /* -------------------------------------------------------------------- */ @@ -2174,31 +2165,41 @@ PyVarObject_HEAD_INIT(NULL, 0) "TreeBuilder", sizeof(TreeBuilderObject), 0, /* methods */ - (destructor)treebuilder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - treebuilder_methods, /* tp_methods */ - 0, /* tp_members */ + (destructor)treebuilder_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + treebuilder_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)treebuilder_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + treebuilder_new, /* tp_new */ + 0, /* tp_free */ }; /* ==================================================================== */ @@ -2684,7 +2685,7 @@ /* setup target handlers */ if (!target) { - target = treebuilder_new(); + target = treebuilder_new(&TreeBuilder_Type, NULL, NULL); if (!target) { EXPAT(ParserFree)(self->parser); PyObject_Del(self->names); @@ -3073,7 +3074,6 @@ static PyMethodDef _functions[] = { {"SubElement", (PyCFunction) subelement, METH_VARARGS|METH_KEYWORDS}, - {"TreeBuilder", (PyCFunction) treebuilder, METH_VARARGS}, #if defined(USE_EXPAT) {"XMLParser", (PyCFunction) xmlparser, METH_VARARGS|METH_KEYWORDS}, #endif @@ -3185,5 +3185,8 @@ Py_INCREF((PyObject *)&Element_Type); PyModule_AddObject(m, "Element", (PyObject *)&Element_Type); + Py_INCREF((PyObject *)&TreeBuilder_Type); + PyModule_AddObject(m, "TreeBuilder", (PyObject *)&TreeBuilder_Type); + return m; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge default -> default): merge heads
by eli.bendersky May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/a3aa056fe8c2 changeset: 77227:a3aa056fe8c2 parent: 77226:717632ae7b3f parent: 77225:c4560d003e36 user: Eli Bendersky <eliben(a)gmail.com> date: Tue May 29 15:45:40 2012 +0300 summary: merge heads files: Doc/library/venv.rst | 9 +-- Doc/using/index.rst | 2 +- Doc/using/scripts.rst | 66 +++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -54,7 +54,7 @@ The command, if run with ``-h``, will show the available options:: usage: pyvenv [-h] [--system-site-packages] [--symlink] [--clear] - ENV_DIR [ENV_DIR ...] + [--upgrade] ENV_DIR [ENV_DIR ...] Creates virtual Python environments in one or more target directories. @@ -69,12 +69,11 @@ --clear Delete the environment directory if it already exists. If not specified and the directory exists, an error is raised. - + --upgrade Upgrade the environment directory to use this version + of Python, assuming Python has been upgraded in-place. If the target directory already exists an error will be raised, unless -the ``--clear`` option was provided, in which case the target -directory will be deleted and virtual environment creation will -proceed as usual. +the ``--clear`` or ``--upgrade`` option was provided. The created ``pyvenv.cfg`` file also includes the ``include-system-site-packages`` key, set to ``true`` if ``venv`` is diff --git a/Doc/using/index.rst b/Doc/using/index.rst --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -17,4 +17,4 @@ unix.rst windows.rst mac.rst - + scripts.rst diff --git a/Doc/using/scripts.rst b/Doc/using/scripts.rst new file mode 100644 --- /dev/null +++ b/Doc/using/scripts.rst @@ -0,0 +1,66 @@ +.. _tools-and-scripts: + +Additional Tools and Scripts +============================ + +pyvenv - Creating virtual environments +-------------------------------------- + +Creation of virtual environments is done by executing the ``pyvenv`` +script:: + + pyvenv /path/to/new/virtual/environment + +Running this command creates the target directory (creating any parent +directories that don't exist already) and places a ``pyvenv.cfg`` file +in it with a ``home`` key pointing to the Python installation the +command was run from. It also creates a ``bin`` (or ``Scripts`` on +Windows) subdirectory containing a copy of the ``python`` binary (or +binaries, in the case of Windows) and the ``pysetup3`` script (to +facilitate easy installation of packages from PyPI into the new virtualenv). +It also creates an (initially empty) ``lib/pythonX.Y/site-packages`` +subdirectory (on Windows, this is ``Lib\site-packages``). + +.. highlight:: none + +On Windows, you may have to invoke the ``pyvenv`` script as follows, if you +don't have the relevant PATH and PATHEXT settings:: + + c:\Temp>c:\Python33\python c:\Python33\Tools\Scripts\pyvenv.py myenv + +or equivalently:: + + c:\Temp>c:\Python33\python -m venv myenv + +The command, if run with ``-h``, will show the available options:: + + usage: pyvenv [-h] [--system-site-packages] [--symlink] [--clear] + [--upgrade] ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + optional arguments: + -h, --help show this help message and exit + --system-site-packages Give access to the global site-packages dir to the + virtual environment. + --symlink Attempt to symlink rather than copy. + --clear Delete the environment directory if it already exists. + If not specified and the directory exists, an error is + raised. + --upgrade Upgrade the environment directory to use this version + of Python, assuming Python has been upgraded in-place. + +If the target directory already exists an error will be raised, unless +the ``--clear`` or ``--upgrade`` option was provided. + +The created ``pyvenv.cfg`` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Multiple paths can be given to ``pyvenv``, in which case an identical +virtualenv will be created, according to the given options, at each +provided path. + -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge default -> default): Merged upstream changes.
by vinay.sajip May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/c4560d003e36 changeset: 77225:c4560d003e36 parent: 77224:ace7c340d95d parent: 77223:22b56b0b8619 user: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: Tue May 29 12:53:00 2012 +0100 summary: Merged upstream changes. files: Doc/library/multiprocessing.rst | 6 +- Include/complexobject.h | 10 +- Include/floatobject.h | 10 +- Include/longobject.h | 18 +- Include/unicodeobject.h | 95 +++- Lib/multiprocessing/__init__.py | 8 +- Lib/multiprocessing/managers.py | 11 +- Lib/multiprocessing/sharedctypes.py | 7 +- Misc/NEWS | 3 + Objects/complexobject.c | 17 +- Objects/floatobject.c | 27 +- Objects/longobject.c | 333 +++++++++--- Objects/stringlib/asciilib.h | 2 +- Objects/stringlib/unicode_format.h | 48 +- Objects/unicodeobject.c | 372 ++++++++++--- Python/formatter_unicode.c | 412 ++++++++------- Python/getargs.c | 5 +- 17 files changed, 912 insertions(+), 472 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -953,7 +953,7 @@ It is possible to create shared objects using shared memory which can be inherited by child processes. -.. function:: Value(typecode_or_type, *args[, lock]) +.. function:: Value(typecode_or_type, *args, lock=True) Return a :mod:`ctypes` object allocated from shared memory. By default the return value is actually a synchronized wrapper for the object. @@ -1045,7 +1045,7 @@ attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *args[, lock]) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1060,7 +1060,7 @@ Note that *lock* is a keyword-only argument. -.. function:: Value(typecode_or_type, *args[, lock]) +.. function:: Value(typecode_or_type, *args, lock=True) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes diff --git a/Include/complexobject.h b/Include/complexobject.h --- a/Include/complexobject.h +++ b/Include/complexobject.h @@ -63,10 +63,12 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyComplex_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif #ifdef __cplusplus diff --git a/Include/floatobject.h b/Include/floatobject.h --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -112,10 +112,12 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Include/longobject.h b/Include/longobject.h --- a/Include/longobject.h +++ b/Include/longobject.h @@ -151,14 +151,22 @@ /* _PyLong_Format: Convert the long to a string object with given base, appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *aa, int base); +PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); + +PyAPI_FUNC(int) _PyLong_FormatWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + int base, + int alternate); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ /* These aren't really part of the long object, but they're handy. The diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -648,8 +648,20 @@ Py_ssize_t from_start, Py_ssize_t how_many ); + +/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so + may crash if parameters are invalid (e.g. if the output string + is too short). */ +PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( + PyObject *to, + Py_ssize_t to_start, + PyObject *from, + Py_ssize_t from_start, + Py_ssize_t how_many + ); #endif +#ifndef Py_LIMITED_API /* Fill a string with a character: write fill_char into unicode[start:start+length]. @@ -658,13 +670,21 @@ Return the number of written character, or return -1 and raise an exception on error. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( PyObject *unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char ); + +/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash + if parameters are invalid (e.g. if length is longer than the string). */ +PyAPI_FUNC(void) _PyUnicode_FastFill( + PyObject *unicode, + Py_ssize_t start, + Py_ssize_t length, + Py_UCS4 fill_char + ); #endif /* Create a Unicode Object from the Py_UNICODE buffer u of the given @@ -696,13 +716,19 @@ const char *u /* UTF-8 encoded string */ ); +#ifndef Py_LIMITED_API /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. Scan the string to find the maximum character. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( int kind, const void *buffer, Py_ssize_t size); + +/* Create a new string from a buffer of ASCII characters. + WARNING: Don't check if the string contains any non-ASCII character. */ +PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( + const char *buffer, + Py_ssize_t size); #endif PyAPI_FUNC(PyObject*) PyUnicode_Substring( @@ -865,12 +891,69 @@ ); #ifndef Py_LIMITED_API +typedef struct { + PyObject *buffer; + void *data; + enum PyUnicode_Kind kind; + Py_UCS4 maxchar; + Py_ssize_t size; + Py_ssize_t pos; + /* minimum length of the buffer when overallocation is enabled, + see _PyUnicodeWriter_Init() */ + Py_ssize_t min_length; + struct { + unsigned char overallocate:1; + /* If readonly is 1, buffer is a shared string (cannot be modified) + and size is set to 0. */ + unsigned char readonly:1; + } flags; +} _PyUnicodeWriter ; + +/* Initialize a Unicode writer. + + If min_length is greater than zero, _PyUnicodeWriter_Prepare() + overallocates the buffer and min_length is the minimum length in characters + of the buffer. */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length); + +/* Prepare the buffer to write 'length' characters + with the specified maximum character. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ + (((MAXCHAR) <= (WRITER)->maxchar \ + && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ + ? 0 \ + : (((LENGTH) == 0) \ + ? 0 \ + : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) + +/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro + instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar); + +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str); + +PyAPI_FUNC(PyObject *) +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); + +PyAPI_FUNC(void) +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); +#endif + +#ifndef Py_LIMITED_API /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py --- a/Lib/multiprocessing/__init__.py +++ b/Lib/multiprocessing/__init__.py @@ -228,19 +228,19 @@ from multiprocessing.sharedctypes import RawArray return RawArray(typecode_or_type, size_or_initializer) -def Value(typecode_or_type, *args, **kwds): +def Value(typecode_or_type, *args, lock=True): ''' Returns a synchronized shared object ''' from multiprocessing.sharedctypes import Value - return Value(typecode_or_type, *args, **kwds) + return Value(typecode_or_type, *args, lock=lock) -def Array(typecode_or_type, size_or_initializer, **kwds): +def Array(typecode_or_type, size_or_initializer, *, lock=True): ''' Returns a synchronized shared array ''' from multiprocessing.sharedctypes import Array - return Array(typecode_or_type, size_or_initializer, **kwds) + return Array(typecode_or_type, size_or_initializer, lock=lock) # # diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1035,12 +1035,11 @@ BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__delslice__', - '__getitem__', '__getslice__', '__len__', '__mul__', - '__reversed__', '__rmul__', '__setitem__', '__setslice__', + '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', + '__mul__', '__reversed__', '__rmul__', '__setitem__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' - )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 + )) class ListProxy(BaseListProxy): def __iadd__(self, value): self._callmethod('extend', (value,)) @@ -1058,8 +1057,8 @@ ArrayProxy = MakeProxyType('ArrayProxy', ( - '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__' - )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 + '__len__', '__getitem__', '__setitem__' + )) PoolProxy = MakeProxyType('PoolProxy', ( diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py --- a/Lib/multiprocessing/sharedctypes.py +++ b/Lib/multiprocessing/sharedctypes.py @@ -63,7 +63,7 @@ result.__init__(*size_or_initializer) return result -def Value(typecode_or_type, *args, lock=None): +def Value(typecode_or_type, *args, lock=True): ''' Return a synchronization wrapper for a Value ''' @@ -76,13 +76,10 @@ raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock) -def Array(typecode_or_type, size_or_initializer, **kwds): +def Array(typecode_or_type, size_or_initializer, *, lock=True): ''' Return a synchronization wrapper for a RawArray ''' - lock = kwds.pop('lock', None) - if kwds: - raise ValueError('unrecognized keyword argument(s): %s' % list(kwds.keys())) obj = RawArray(typecode_or_type, size_or_initializer) if lock is False: return obj diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ - Issue #14835: Make plistlib output empty arrays & dicts like OS X. Patch by Sidney San Martín. +- Issue #14744: Use the new _PyUnicodeWriter internal API to speed up + str%args and str.format(args). + - Issue #14930: Make memoryview objects weakrefable. - Issue #14775: Fix a potential quadratic dict build-up due to the garbage diff --git a/Objects/complexobject.c b/Objects/complexobject.c --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -699,11 +699,22 @@ complex__format__(PyObject* self, PyObject* args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) - return NULL; - return _PyComplex_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + return NULL; + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyComplex_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } #if 0 diff --git a/Objects/floatobject.c b/Objects/floatobject.c --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -267,13 +267,15 @@ float_repr(PyFloatObject *v) { PyObject *result; - char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), - 'r', 0, - Py_DTSF_ADD_DOT_0, - NULL); + char *buf; + + buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), + 'r', 0, + Py_DTSF_ADD_DOT_0, + NULL); if (!buf) return PyErr_NoMemory(); - result = PyUnicode_FromString(buf); + result = _PyUnicode_FromASCII(buf, strlen(buf)); PyMem_Free(buf); return result; } @@ -1703,11 +1705,22 @@ float__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyFloat_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(float__format__doc, diff --git a/Objects/longobject.c b/Objects/longobject.c --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1550,20 +1550,22 @@ string. (Return value is non-shared so that callers can modify the returned value if necessary.) */ -static PyObject * -long_to_decimal_string(PyObject *aa) +static int +long_to_decimal_string_internal(PyObject *aa, + PyObject **p_output, + _PyUnicodeWriter *writer) { PyLongObject *scratch, *a; PyObject *str; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; - unsigned char *p; int negative; + enum PyUnicode_Kind kind; a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); negative = Py_SIZE(a) < 0; @@ -1580,13 +1582,13 @@ if (size_a > PY_SSIZE_T_MAX / PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "long is too large to format"); - return NULL; + return -1; } /* the expression size_a * PyLong_SHIFT is now safe from overflow */ size = 1 + size_a * PyLong_SHIFT / (3 * _PyLong_DECIMAL_SHIFT); scratch = _PyLong_New(size); if (scratch == NULL) - return NULL; + return -1; /* convert array of base _PyLong_BASE digits in pin to an array of base _PyLong_DECIMAL_BASE digits in pout, following Knuth (TAOCP, @@ -1609,7 +1611,7 @@ /* check for keyboard interrupt */ SIGCHECK({ Py_DECREF(scratch); - return NULL; + return -1; }); } /* pout should have at least one digit, so that the case when a = 0 @@ -1625,65 +1627,113 @@ tenpow *= 10; strlen++; } - str = PyUnicode_New(strlen, '9'); - if (str == NULL) { - Py_DECREF(scratch); + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, strlen, '9') == -1) + return -1; + kind = writer->kind; + str = NULL; + } + else { + str = PyUnicode_New(strlen, '9'); + if (str == NULL) { + Py_DECREF(scratch); + return -1; + } + kind = PyUnicode_KIND(str); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ + else \ + p = (TYPE*)PyUnicode_DATA(str) + strlen; \ + \ + *p = '\0'; \ + /* pout[0] through pout[size-2] contribute exactly \ + _PyLong_DECIMAL_SHIFT digits each */ \ + for (i=0; i < size - 1; i++) { \ + rem = pout[i]; \ + for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } \ + } \ + /* pout[size-1]: always produce at least one decimal digit */ \ + rem = pout[i]; \ + do { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } while (rem != 0); \ + \ + /* and sign */ \ + if (negative) \ + *--p = '-'; \ + \ + /* check we've counted correctly */ \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(str)); \ + } while (0) + + /* fill the string right-to-left */ + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS + + Py_DECREF(scratch); + if (writer) { + writer->pos += strlen; + } + else { + assert(_PyUnicode_CheckConsistency(str, 1)); + *p_output = (PyObject *)str; + } + return 0; +} + +static PyObject * +long_to_decimal_string(PyObject *aa) +{ + PyObject *v; + if (long_to_decimal_string_internal(aa, &v, NULL) == -1) return NULL; - } - - /* fill the string right-to-left */ - assert(PyUnicode_KIND(str) == PyUnicode_1BYTE_KIND); - p = PyUnicode_1BYTE_DATA(str) + strlen; - *p = '\0'; - /* pout[0] through pout[size-2] contribute exactly - _PyLong_DECIMAL_SHIFT digits each */ - for (i=0; i < size - 1; i++) { - rem = pout[i]; - for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { - *--p = '0' + rem % 10; - rem /= 10; - } - } - /* pout[size-1]: always produce at least one decimal digit */ - rem = pout[i]; - do { - *--p = '0' + rem % 10; - rem /= 10; - } while (rem != 0); - - /* and sign */ - if (negative) - *--p = '-'; - - /* check we've counted correctly */ - assert(p == PyUnicode_1BYTE_DATA(str)); - assert(_PyUnicode_CheckConsistency(str, 1)); - Py_DECREF(scratch); - return (PyObject *)str; + return v; } /* Convert a long int object to a string, using a given conversion base, - which should be one of 2, 8, 10 or 16. Return a string object. - If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x'. */ - -PyObject * -_PyLong_Format(PyObject *aa, int base) + which should be one of 2, 8 or 16. Return a string object. + If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x' + if alternate is nonzero. */ + +static int +long_format_binary(PyObject *aa, int base, int alternate, + PyObject **p_output, _PyUnicodeWriter *writer) { register PyLongObject *a = (PyLongObject *)aa; PyObject *v; Py_ssize_t sz; Py_ssize_t size_a; - Py_UCS1 *p; + enum PyUnicode_Kind kind; int negative; int bits; - assert(base == 2 || base == 8 || base == 10 || base == 16); - if (base == 10) - return long_to_decimal_string((PyObject *)a); - + assert(base == 2 || base == 8 || base == 16); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); negative = Py_SIZE(a) < 0; @@ -1706,7 +1756,7 @@ /* Compute exact length 'sz' of output string. */ if (size_a == 0) { - sz = 3; + sz = 1; } else { Py_ssize_t size_a_in_bits; @@ -1714,56 +1764,126 @@ if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "int is too large to format"); - return NULL; + return -1; } size_a_in_bits = (size_a - 1) * PyLong_SHIFT + bits_in_digit(a->ob_digit[size_a - 1]); - /* Allow 2 characters for prefix and 1 for a '-' sign. */ - sz = 2 + negative + (size_a_in_bits + (bits - 1)) / bits; - } - - v = PyUnicode_New(sz, 'x'); - if (v == NULL) { + /* Allow 1 character for a '-' sign. */ + sz = negative + (size_a_in_bits + (bits - 1)) / bits; + } + if (alternate) { + /* 2 characters for prefix */ + sz += 2; + } + + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) + return -1; + kind = writer->kind; + v = NULL; + } + else { + v = PyUnicode_New(sz, 'x'); + if (v == NULL) + return -1; + kind = PyUnicode_KIND(v); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ + else \ + p = (TYPE*)PyUnicode_DATA(v) + sz; \ + \ + if (size_a == 0) { \ + *--p = '0'; \ + } \ + else { \ + /* JRH: special case for power-of-2 bases */ \ + twodigits accum = 0; \ + int accumbits = 0; /* # of bits in accum */ \ + Py_ssize_t i; \ + for (i = 0; i < size_a; ++i) { \ + accum |= (twodigits)a->ob_digit[i] << accumbits; \ + accumbits += PyLong_SHIFT; \ + assert(accumbits >= bits); \ + do { \ + char cdigit; \ + cdigit = (char)(accum & (base - 1)); \ + cdigit += (cdigit < 10) ? '0' : 'a'-10; \ + *--p = cdigit; \ + accumbits -= bits; \ + accum >>= bits; \ + } while (i < size_a-1 ? accumbits >= bits : accum > 0); \ + } \ + } \ + \ + if (alternate) { \ + if (base == 16) \ + *--p = 'x'; \ + else if (base == 8) \ + *--p = 'o'; \ + else /* (base == 2) */ \ + *--p = 'b'; \ + *--p = '0'; \ + } \ + if (negative) \ + *--p = '-'; \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(v)); \ + } while (0) + + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS + + if (writer) { + writer->pos += sz; + } + else { + assert(_PyUnicode_CheckConsistency(v, 1)); + *p_output = v; + } + return 0; +} + +PyObject * +_PyLong_Format(PyObject *obj, int base) +{ + PyObject *str; + int err; + if (base == 10) + err = long_to_decimal_string_internal(obj, &str, NULL); + else + err = long_format_binary(obj, base, 1, &str, NULL); + if (err == -1) return NULL; - } - assert(PyUnicode_KIND(v) == PyUnicode_1BYTE_KIND); - - p = PyUnicode_1BYTE_DATA(v) + sz; - if (size_a == 0) { - *--p = '0'; - } - else { - /* JRH: special case for power-of-2 bases */ - twodigits accum = 0; - int accumbits = 0; /* # of bits in accum */ - Py_ssize_t i; - for (i = 0; i < size_a; ++i) { - accum |= (twodigits)a->ob_digit[i] << accumbits; - accumbits += PyLong_SHIFT; - assert(accumbits >= bits); - do { - char cdigit; - cdigit = (char)(accum & (base - 1)); - cdigit += (cdigit < 10) ? '0' : 'a'-10; - *--p = cdigit; - accumbits -= bits; - accum >>= bits; - } while (i < size_a-1 ? accumbits >= bits : accum > 0); - } - } - - if (base == 16) - *--p = 'x'; - else if (base == 8) - *--p = 'o'; - else /* (base == 2) */ - *--p = 'b'; - *--p = '0'; - if (negative) - *--p = '-'; - assert(p == PyUnicode_1BYTE_DATA(v)); - assert(_PyUnicode_CheckConsistency(v, 1)); - return v; + return str; +} + +int +_PyLong_FormatWriter(_PyUnicodeWriter *writer, + PyObject *obj, + int base, int alternate) +{ + if (base == 10) + return long_to_decimal_string_internal(obj, NULL, writer); + else + return long_format_binary(obj, base, alternate, NULL, writer); } /* Table of digit values for 8-bit string -> integer conversion. @@ -4232,11 +4352,22 @@ long__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyLong_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyLong_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } /* Return a pair (q, r) such that a = b * q + r, and diff --git a/Objects/stringlib/asciilib.h b/Objects/stringlib/asciilib.h --- a/Objects/stringlib/asciilib.h +++ b/Objects/stringlib/asciilib.h @@ -18,7 +18,7 @@ #define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL #define STRINGLIB_STR PyUnicode_1BYTE_DATA #define STRINGLIB_LEN PyUnicode_GET_LENGTH -#define STRINGLIB_NEW unicode_fromascii +#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((char*)(STR),(LEN)) #define STRINGLIB_RESIZE not_supported #define STRINGLIB_CHECK PyUnicode_Check #define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -499,26 +499,26 @@ int ok = 0; PyObject *result = NULL; PyObject *format_spec_object = NULL; - PyObject *(*formatter)(PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; - Py_ssize_t len; + int (*formatter) (_PyUnicodeWriter*, PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; + int err; /* If we know the type exactly, skip the lookup of __format__ and just call the formatter directly. */ if (PyUnicode_CheckExact(fieldobj)) - formatter = _PyUnicode_FormatAdvanced; + formatter = _PyUnicode_FormatAdvancedWriter; else if (PyLong_CheckExact(fieldobj)) - formatter =_PyLong_FormatAdvanced; + formatter = _PyLong_FormatAdvancedWriter; else if (PyFloat_CheckExact(fieldobj)) - formatter = _PyFloat_FormatAdvanced; - - /* XXX: for 2.6, convert format_spec to the appropriate type - (unicode, str) */ + formatter = _PyFloat_FormatAdvancedWriter; + else if (PyComplex_CheckExact(fieldobj)) + formatter = _PyComplex_FormatAdvancedWriter; if (formatter) { /* we know exactly which formatter will be called when __format__ is looked up, so call it directly, instead. */ - result = formatter(fieldobj, format_spec->str, - format_spec->start, format_spec->end); + err = formatter(writer, fieldobj, format_spec->str, + format_spec->start, format_spec->end); + return (err == 0); } else { /* We need to create an object out of the pointers we have, because @@ -536,17 +536,11 @@ } if (result == NULL) goto done; - if (PyUnicode_READY(result) == -1) + + if (_PyUnicodeWriter_WriteStr(writer, result) == -1) goto done; + ok = 1; - len = PyUnicode_GET_LENGTH(result); - if (_PyUnicodeWriter_Prepare(writer, - len, PyUnicode_MAX_CHAR_VALUE(result)) == -1) - goto done; - copy_characters(writer->buffer, writer->pos, - result, 0, len); - writer->pos += len; - ok = 1; done: Py_XDECREF(format_spec_object); Py_XDECREF(result); @@ -897,16 +891,19 @@ err = _PyUnicodeWriter_Prepare(writer, sublen, maxchar); if (err == -1) return 0; - copy_characters(writer->buffer, writer->pos, - literal.str, literal.start, sublen); + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + literal.str, literal.start, sublen); writer->pos += sublen; } - if (field_present) + if (field_present) { + if (iter.str.start == iter.str.end) + writer->flags.overallocate = 0; if (!output_markup(&field_name, &format_spec, format_spec_needs_expanding, conversion, writer, args, kwargs, recursion_depth, auto_number)) return 0; + } } return result; } @@ -921,7 +918,7 @@ int recursion_depth, AutoNumber *auto_number) { _PyUnicodeWriter writer; - Py_ssize_t initlen; + Py_ssize_t minlen; /* check the recursion level */ if (recursion_depth <= 0) { @@ -930,9 +927,8 @@ return NULL; } - initlen = PyUnicode_GET_LENGTH(input->str) + 100; - if (_PyUnicodeWriter_Init(&writer, initlen, 127) == -1) - return NULL; + minlen = PyUnicode_GET_LENGTH(input->str) + 100; + _PyUnicodeWriter_Init(&writer, minlen); if (!do_markup(input, args, kwargs, &writer, recursion_depth, auto_number)) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -225,16 +225,10 @@ /* forward */ static PyUnicodeObject *_PyUnicode_New(Py_ssize_t length); static PyObject* get_latin1_char(unsigned char ch); -static void copy_characters( - PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many); static int unicode_modifiable(PyObject *unicode); static PyObject * -unicode_fromascii(const unsigned char *s, Py_ssize_t size); -static PyObject * _PyUnicode_FromUCS1(const unsigned char *s, Py_ssize_t size); static PyObject * _PyUnicode_FromUCS2(const Py_UCS2 *s, Py_ssize_t size); @@ -783,7 +777,7 @@ return NULL; copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode)); - copy_characters(copy, 0, unicode, 0, copy_length); + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length); return copy; } else { @@ -1154,15 +1148,16 @@ assert(0 <= from_start); assert(0 <= to_start); assert(PyUnicode_Check(from)); - assert(PyUnicode_Check(to)); assert(PyUnicode_IS_READY(from)); - assert(PyUnicode_IS_READY(to)); assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); - assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); if (how_many == 0) return 0; + assert(PyUnicode_Check(to)); + assert(PyUnicode_IS_READY(to)); + assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); + from_kind = PyUnicode_KIND(from); from_data = PyUnicode_DATA(from); to_kind = PyUnicode_KIND(to); @@ -1267,10 +1262,10 @@ return 0; } -static void -copy_characters(PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many) +void +_PyUnicode_FastCopyCharacters( + PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) { (void)_copy_characters(to, to_start, from, from_start, how_many, 0); } @@ -1292,6 +1287,14 @@ if (PyUnicode_READY(to) == -1) return -1; + if (from_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + if (to_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } how_many = Py_MIN(PyUnicode_GET_LENGTH(from), how_many); if (to_start + how_many > PyUnicode_GET_LENGTH(to)) { PyErr_Format(PyExc_SystemError, @@ -1641,7 +1644,7 @@ maxchar); if (result == NULL) return -1; - PyUnicode_CopyCharacters(result, 0, *p_unicode, 0, length); + _PyUnicode_FastCopyCharacters(result, 0, *p_unicode, 0, length); Py_DECREF(*p_unicode); *p_unicode = result; return 0; @@ -1841,9 +1844,10 @@ /* Internal function, doesn't check maximum character */ -static PyObject* -unicode_fromascii(const unsigned char* s, Py_ssize_t size) -{ +PyObject* +_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size) +{ + const unsigned char *s = (const unsigned char *)buffer; PyObject *unicode; if (size == 1) { #ifdef Py_DEBUG @@ -2085,7 +2089,7 @@ return; } copy = PyUnicode_New(len, max_char); - copy_characters(copy, 0, unicode, 0, len); + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len); Py_DECREF(unicode); *p_unicode = copy; } @@ -2753,7 +2757,7 @@ (void) va_arg(vargs, char *); size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -2767,7 +2771,7 @@ Py_ssize_t size; assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); size = PyUnicode_GET_LENGTH(obj); - copy_characters(string, i, obj, 0, size); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); i += size; break; } @@ -2779,13 +2783,13 @@ if (obj) { size = PyUnicode_GET_LENGTH(obj); assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); - copy_characters(string, i, obj, 0, size); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); i += size; } else { size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; Py_DECREF(*callresult); } @@ -2800,7 +2804,7 @@ /* unused, since we already have the result */ (void) va_arg(vargs, PyObject *); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -4171,7 +4175,7 @@ if (unicode_widen(output, *outpos, PyUnicode_MAX_CHAR_VALUE(repunicode)) < 0) goto onError; - copy_characters(*output, *outpos, repunicode, 0, replen); + _PyUnicode_FastCopyCharacters(*output, *outpos, repunicode, 0, replen); *outpos += replen; } else { @@ -9216,12 +9220,14 @@ /* If the maxchar increased so that the kind changed, not all characters are representable anymore and we need to fix the string again. This only happens in very few cases. */ - copy_characters(v, 0, self, 0, PyUnicode_GET_LENGTH(self)); + _PyUnicode_FastCopyCharacters(v, 0, + self, 0, PyUnicode_GET_LENGTH(self)); maxchar_old = fixfct(v); assert(maxchar_old > 0 && maxchar_old <= maxchar_new); } else { - copy_characters(v, 0, u, 0, PyUnicode_GET_LENGTH(self)); + _PyUnicode_FastCopyCharacters(v, 0, + u, 0, PyUnicode_GET_LENGTH(self)); } Py_DECREF(u); assert(_PyUnicode_CheckConsistency(v, 1)); @@ -9603,7 +9609,7 @@ res_data += kind * seplen; } else { - copy_characters(res, res_offset, sep, 0, seplen); + _PyUnicode_FastCopyCharacters(res, res_offset, sep, 0, seplen); res_offset += seplen; } } @@ -9616,7 +9622,7 @@ res_data += kind * itemlen; } else { - copy_characters(res, res_offset, item, 0, itemlen); + _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); res_offset += itemlen; } } @@ -9663,13 +9669,25 @@ } \ } while (0) +void +_PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, + Py_UCS4 fill_char) +{ + const enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + assert(PyUnicode_IS_READY(unicode)); + assert(unicode_modifiable(unicode)); + assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); + assert(start >= 0); + assert(start + length <= PyUnicode_GET_LENGTH(unicode)); + FILL(kind, data, fill_char, start, length); +} + Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char) { Py_ssize_t maxlen; - enum PyUnicode_Kind kind; - void *data; if (!PyUnicode_Check(unicode)) { PyErr_BadInternalCall(); @@ -9680,6 +9698,10 @@ if (unicode_check_modifiable(unicode)) return -1; + if (start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } if (fill_char > PyUnicode_MAX_CHAR_VALUE(unicode)) { PyErr_SetString(PyExc_ValueError, "fill character is bigger than " @@ -9692,9 +9714,7 @@ if (length <= 0) return 0; - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - FILL(kind, data, fill_char, start, length); + _PyUnicode_FastFill(unicode, start, length, fill_char); return length; } @@ -9734,7 +9754,7 @@ FILL(kind, data, fill, 0, left); if (right) FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); - copy_characters(u, left, self, 0, _PyUnicode_LENGTH(self)); + _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); assert(_PyUnicode_CheckConsistency(u, 1)); return u; } @@ -10058,7 +10078,7 @@ u = PyUnicode_New(slen, maxchar); if (!u) goto error; - copy_characters(u, 0, self, 0, slen); + _PyUnicode_FastCopyCharacters(u, 0, self, 0, slen); rkind = PyUnicode_KIND(u); PyUnicode_WRITE(rkind, PyUnicode_DATA(u), pos, u2); @@ -10626,8 +10646,8 @@ w = PyUnicode_New(new_len, maxchar); if (w == NULL) goto onError; - copy_characters(w, 0, u, 0, u_len); - copy_characters(w, u_len, v, 0, v_len); + _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); + _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); Py_DECREF(u); Py_DECREF(v); assert(_PyUnicode_CheckConsistency(w, 1)); @@ -10702,7 +10722,7 @@ goto error; } /* copy 'right' into the newly allocated area of 'left' */ - copy_characters(*p_left, left_len, right, 0, right_len); + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); } else { maxchar = PyUnicode_MAX_CHAR_VALUE(left); @@ -10713,8 +10733,8 @@ res = PyUnicode_New(new_len, maxchar); if (res == NULL) goto error; - copy_characters(res, 0, left, 0, left_len); - copy_characters(res, left_len, right, 0, right_len); + _PyUnicode_FastCopyCharacters(res, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(res, left_len, right, 0, right_len); Py_DECREF(left); *p_left = res; } @@ -11650,7 +11670,7 @@ length = end - start; if (PyUnicode_IS_ASCII(self)) { data = PyUnicode_1BYTE_DATA(self); - return unicode_fromascii(data + start, length); + return _PyUnicode_FromASCII((char*)(data + start), length); } else { kind = PyUnicode_KIND(self); @@ -12769,60 +12789,74 @@ return PyBool_FromLong(result); } -typedef struct { - PyObject *buffer; - void *data; - enum PyUnicode_Kind kind; - Py_UCS4 maxchar; - Py_ssize_t pos; -} _PyUnicodeWriter ; - Py_LOCAL_INLINE(void) _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { + writer->size = PyUnicode_GET_LENGTH(writer->buffer); writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); writer->data = PyUnicode_DATA(writer->buffer); writer->kind = PyUnicode_KIND(writer->buffer); } -Py_LOCAL(int) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) -{ - writer->pos = 0; - writer->buffer = PyUnicode_New(length, maxchar); - if (writer->buffer == NULL) - return -1; - _PyUnicodeWriter_Update(writer); - return 0; -} - -Py_LOCAL_INLINE(int) -_PyUnicodeWriter_Prepare(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) +void +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length) +{ + memset(writer, 0, sizeof(*writer)); +#ifdef Py_DEBUG + writer->kind = 5; /* invalid kind */ +#endif + writer->min_length = Py_MAX(min_length, 100); + writer->flags.overallocate = (min_length > 0); +} + +int +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar) { Py_ssize_t newlen; PyObject *newbuffer; + assert(length > 0); + if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); return -1; } newlen = writer->pos + length; - if (newlen > PyUnicode_GET_LENGTH(writer->buffer)) { - /* overallocate 25% to limit the number of resize */ - if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) - newlen += newlen / 4; - - if (maxchar > writer->maxchar) { + if (writer->buffer == NULL) { + if (writer->flags.overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + writer->buffer = PyUnicode_New(newlen, maxchar); + if (writer->buffer == NULL) + return -1; + _PyUnicodeWriter_Update(writer); + return 0; + } + + if (newlen > writer->size) { + if (writer->flags.overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + + if (maxchar > writer->maxchar || writer->flags.readonly) { /* resize + widen */ newbuffer = PyUnicode_New(newlen, maxchar); if (newbuffer == NULL) return -1; - PyUnicode_CopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); Py_DECREF(writer->buffer); + writer->flags.readonly = 0; } else { newbuffer = resize_compact(writer->buffer, newlen); @@ -12833,25 +12867,76 @@ _PyUnicodeWriter_Update(writer); } else if (maxchar > writer->maxchar) { - if (unicode_widen(&writer->buffer, writer->pos, maxchar) < 0) + assert(!writer->flags.readonly); + newbuffer = PyUnicode_New(writer->size, maxchar); + if (newbuffer == NULL) return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_DECREF(writer->buffer); + writer->buffer = newbuffer; _PyUnicodeWriter_Update(writer); } return 0; } -Py_LOCAL(PyObject *) +int +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) +{ + Py_UCS4 maxchar; + Py_ssize_t len; + + if (PyUnicode_READY(str) == -1) + return -1; + len = PyUnicode_GET_LENGTH(str); + if (len == 0) + return 0; + maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { + if (writer->buffer == NULL && !writer->flags.overallocate) { + Py_INCREF(str); + writer->buffer = str; + _PyUnicodeWriter_Update(writer); + writer->flags.readonly = 1; + writer->size = 0; + writer->pos += len; + return 0; + } + if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) + return -1; + } + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, 0, len); + writer->pos += len; + return 0; +} + +PyObject * _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) { - if (PyUnicode_Resize(&writer->buffer, writer->pos) < 0) { - Py_DECREF(writer->buffer); - return NULL; + if (writer->pos == 0) { + Py_XDECREF(writer->buffer); + Py_INCREF(unicode_empty); + return unicode_empty; + } + if (writer->flags.readonly) { + assert(PyUnicode_GET_LENGTH(writer->buffer) == writer->pos); + return writer->buffer; + } + if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { + PyObject *newbuffer; + newbuffer = resize_compact(writer->buffer, writer->pos); + if (newbuffer == NULL) { + Py_DECREF(writer->buffer); + return NULL; + } + writer->buffer = newbuffer; } assert(_PyUnicode_CheckConsistency(writer->buffer, 1)); return writer->buffer; } -Py_LOCAL(void) +void _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) { Py_CLEAR(writer->buffer); @@ -12874,14 +12959,24 @@ static PyObject * unicode__format__(PyObject* self, PyObject* args) { - PyObject *format_spec, *out; + PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - out = _PyUnicode_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); - return out; + if (PyUnicode_READY(self) == -1) + return NULL; + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyUnicode_FormatAdvancedWriter(&writer, + self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(p_format__doc__, @@ -13111,16 +13206,17 @@ /* Returns a new reference to a PyUnicode object, or NULL on failure. */ -static PyObject * -formatfloat(PyObject *v, int flags, int prec, int type) +static int +formatfloat(PyObject *v, int flags, int prec, int type, + PyObject **p_output, _PyUnicodeWriter *writer) { char *p; - PyObject *result; double x; + Py_ssize_t len; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) - return NULL; + return -1; if (prec < 0) prec = 6; @@ -13128,10 +13224,20 @@ p = PyOS_double_to_string(x, type, prec, (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL); if (p == NULL) - return NULL; - result = unicode_fromascii((unsigned char*)p, strlen(p)); + return -1; + len = strlen(p); + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) + return -1; + memcpy(writer->data + writer->pos * writer->kind, + p, + len); + writer->pos += len; + } + else + *p_output = _PyUnicode_FromASCII(p, len); PyMem_Free(p); - return result; + return 0; } /* formatlong() emulates the format codes d, u, o, x and X, and @@ -13267,7 +13373,7 @@ } if (!PyUnicode_Check(result) || len != PyUnicode_GET_LENGTH(result)) { PyObject *unicode; - unicode = unicode_fromascii((unsigned char *)buf, len); + unicode = _PyUnicode_FromASCII(buf, len); Py_DECREF(result); result = unicode; } @@ -13336,8 +13442,7 @@ fmtcnt = PyUnicode_GET_LENGTH(uformat); fmtpos = 0; - if (_PyUnicodeWriter_Init(&writer, fmtcnt + 100, 127) < 0) - goto onError; + _PyUnicodeWriter_Init(&writer, fmtcnt + 100); if (PyTuple_Check(args)) { arglen = PyTuple_Size(args); @@ -13368,8 +13473,8 @@ if (_PyUnicodeWriter_Prepare(&writer, sublen, maxchar) == -1) goto onError; - copy_characters(writer.buffer, writer.pos, - uformat, nonfmtpos, sublen); + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + uformat, nonfmtpos, sublen); writer.pos += sublen; } else { @@ -13530,6 +13635,8 @@ "incomplete format"); goto onError; } + if (fmtcnt == 0) + writer.flags.overallocate = 0; if (c == '%') { if (_PyUnicodeWriter_Prepare(&writer, 1, '%') == -1) @@ -13539,7 +13646,6 @@ continue; } - v = getnextarg(args, arglen, &argidx); if (v == NULL) goto onError; @@ -13552,6 +13658,13 @@ case 's': case 'r': case 'a': + if (PyLong_CheckExact(v) && width == -1 && prec == -1) { + /* Fast path */ + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + } + if (PyUnicode_CheckExact(v) && c == 's') { temp = v; Py_INCREF(temp); @@ -13572,6 +13685,32 @@ case 'o': case 'x': case 'X': + if (PyLong_CheckExact(v) + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + switch(c) + { + case 'd': + case 'i': + case 'u': + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'x': + if (_PyLong_FormatWriter(&writer, v, 16, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'o': + if (_PyLong_FormatWriter(&writer, v, 8, flags & F_ALT) == -1) + goto onError; + goto nextarg; + default: + break; + } + } + isnumok = 0; if (PyNumber_Check(v)) { PyObject *iobj=NULL; @@ -13611,10 +13750,20 @@ case 'F': case 'g': case 'G': + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (formatfloat(v, flags, prec, c, NULL, &writer) == -1) + goto onError; + goto nextarg; + } + sign = 1; if (flags & F_ZERO) fill = '0'; - temp = formatfloat(v, flags, prec, c); + if (formatfloat(v, flags, prec, c, &temp, NULL) == -1) + temp = NULL; break; case 'c': @@ -13622,6 +13771,14 @@ Py_UCS4 ch = formatchar(v); if (ch == (Py_UCS4) -1) goto onError; + if (width == -1 && prec == -1) { + /* Fast path */ + if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1) + goto onError; + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch); + writer.pos += 1; + goto nextarg; + } temp = PyUnicode_FromOrdinal(ch); break; } @@ -13638,6 +13795,16 @@ if (temp == NULL) goto onError; assert (PyUnicode_Check(temp)); + + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (_PyUnicodeWriter_WriteStr(&writer, temp) == -1) + goto onError; + goto nextarg; + } + if (PyUnicode_READY(temp) == -1) { Py_CLEAR(temp); goto onError; @@ -13676,15 +13843,15 @@ if (!(flags & F_LJUST)) { if (sign) { if ((width-1) > len) - bufmaxchar = Py_MAX(bufmaxchar, fill); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } else { if (width > len) - bufmaxchar = Py_MAX(bufmaxchar, fill); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } } maxchar = _PyUnicode_FindMaxChar(temp, 0, pindex+len); - bufmaxchar = Py_MAX(bufmaxchar, maxchar); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar); buflen = width; if (sign && len == width) @@ -13737,8 +13904,8 @@ } } - copy_characters(writer.buffer, writer.pos, - temp, pindex, len); + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + temp, pindex, len); writer.pos += len; if (width > len) { sublen = width - len; @@ -13746,6 +13913,7 @@ writer.pos += sublen; } +nextarg: if (dict && (argidx < arglen) && c != '%') { PyErr_SetString(PyExc_TypeError, "not all arguments converted during string formatting"); diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -316,21 +316,28 @@ /* Do the padding, and return a pointer to where the caller-supplied content goes. */ static Py_ssize_t -fill_padding(PyObject *s, Py_ssize_t start, Py_ssize_t nchars, +fill_padding(_PyUnicodeWriter *writer, + Py_ssize_t nchars, Py_UCS4 fill_char, Py_ssize_t n_lpadding, Py_ssize_t n_rpadding) { + Py_ssize_t pos; + /* Pad on left. */ - if (n_lpadding) - PyUnicode_Fill(s, start, start + n_lpadding, fill_char); + if (n_lpadding) { + pos = writer->pos; + _PyUnicode_FastFill(writer->buffer, pos, n_lpadding, fill_char); + } /* Pad on right. */ - if (n_rpadding) - PyUnicode_Fill(s, start + nchars + n_lpadding, - start + nchars + n_lpadding + n_rpadding, fill_char); + if (n_rpadding) { + pos = writer->pos + nchars + n_lpadding; + _PyUnicode_FastFill(writer->buffer, pos, n_rpadding, fill_char); + } /* Pointer to the user content. */ - return start + n_lpadding; + writer->pos += n_lpadding; + return 0; } /************************************************************************/ @@ -541,7 +548,7 @@ as determined in calc_number_widths(). Return -1 on error, or 0 on success. */ static int -fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec, +fill_number(_PyUnicodeWriter *writer, const NumberFieldWidths *spec, PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end, PyObject *prefix, Py_ssize_t p_start, Py_UCS4 fill_char, @@ -549,36 +556,38 @@ { /* Used to keep track of digits, decimal, and remainder. */ Py_ssize_t d_pos = d_start; - unsigned int kind = PyUnicode_KIND(out); - void *data = PyUnicode_DATA(out); + const enum PyUnicode_Kind kind = writer->kind; + const void *data = writer->data; Py_ssize_t r; if (spec->n_lpadding) { - PyUnicode_Fill(out, pos, pos + spec->n_lpadding, fill_char); - pos += spec->n_lpadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_lpadding, fill_char); + writer->pos += spec->n_lpadding; } if (spec->n_sign == 1) { - PyUnicode_WRITE(kind, data, pos++, spec->sign); + PyUnicode_WRITE(kind, data, writer->pos, spec->sign); + writer->pos++; } if (spec->n_prefix) { - if (PyUnicode_CopyCharacters(out, pos, - prefix, p_start, - spec->n_prefix) < 0) - return -1; + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + prefix, p_start, + spec->n_prefix); if (toupper) { Py_ssize_t t; for (t = 0; t < spec->n_prefix; t++) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + Py_UCS4 c = PyUnicode_READ(kind, data, writer->pos + t); c = Py_TOUPPER(c); assert (c <= 127); - PyUnicode_WRITE(kind, data, pos + t, c); + PyUnicode_WRITE(kind, data, writer->pos + t, c); } } - pos += spec->n_prefix; + writer->pos += spec->n_prefix; } if (spec->n_spadding) { - PyUnicode_Fill(out, pos, pos + spec->n_spadding, fill_char); - pos += spec->n_spadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_spadding, fill_char); + writer->pos += spec->n_spadding; } /* Only for type 'c' special case, it has no digits. */ @@ -594,7 +603,7 @@ return -1; } r = _PyUnicode_InsertThousandsGrouping( - out, pos, + writer->buffer, writer->pos, spec->n_grouped_digits, pdigits + kind * d_pos, spec->n_digits, spec->n_min_width, @@ -609,34 +618,38 @@ if (toupper) { Py_ssize_t t; for (t = 0; t < spec->n_grouped_digits; t++) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + Py_UCS4 c = PyUnicode_READ(kind, data, writer->pos + t); c = Py_TOUPPER(c); if (c > 127) { PyErr_SetString(PyExc_SystemError, "non-ascii grouped digit"); return -1; } - PyUnicode_WRITE(kind, data, pos + t, c); + PyUnicode_WRITE(kind, data, writer->pos + t, c); } } - pos += spec->n_grouped_digits; + writer->pos += spec->n_grouped_digits; if (spec->n_decimal) { - if (PyUnicode_CopyCharacters(out, pos, locale->decimal_point, 0, spec->n_decimal) < 0) - return -1; - pos += spec->n_decimal; + _PyUnicode_FastCopyCharacters( + writer->buffer, writer->pos, + locale->decimal_point, 0, spec->n_decimal); + writer->pos += spec->n_decimal; d_pos += 1; } if (spec->n_remainder) { - if (PyUnicode_CopyCharacters(out, pos, digits, d_pos, spec->n_remainder) < 0) - return -1; - pos += spec->n_remainder; + _PyUnicode_FastCopyCharacters( + writer->buffer, writer->pos, + digits, d_pos, spec->n_remainder); + writer->pos += spec->n_remainder; d_pos += spec->n_remainder; } if (spec->n_rpadding) { - PyUnicode_Fill(out, pos, pos + spec->n_rpadding, fill_char); - pos += spec->n_rpadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_rpadding, + fill_char); + writer->pos += spec->n_rpadding; } return 0; } @@ -707,17 +720,20 @@ /*********** string formatting ******************************************/ /************************************************************************/ -static PyObject * -format_string_internal(PyObject *value, const InternalFormatSpec *format) +static int +format_string_internal(PyObject *value, const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { Py_ssize_t lpad; Py_ssize_t rpad; Py_ssize_t total; - Py_ssize_t pos; - Py_ssize_t len = PyUnicode_GET_LENGTH(value); - PyObject *result = NULL; + Py_ssize_t len; + int result = -1; Py_UCS4 maxchar; + assert(PyUnicode_IS_READY(value)); + len = PyUnicode_GET_LENGTH(value); + /* sign is not allowed on strings */ if (format->sign != '\0') { PyErr_SetString(PyExc_ValueError, @@ -741,6 +757,11 @@ goto done; } + if (format->width == -1 && format->precision == -1) { + /* Fast path */ + return _PyUnicodeWriter_WriteStr(writer, value); + } + /* if precision is specified, output no more that format.precision characters */ if (format->precision >= 0 && len >= format->precision) { @@ -754,21 +775,23 @@ maxchar = Py_MAX(maxchar, format->fill_char); /* allocate the resulting string */ - result = PyUnicode_New(total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, total, maxchar) == -1) goto done; /* Write into that space. First the padding. */ - pos = fill_padding(result, 0, len, - format->fill_char=='\0'?' ':format->fill_char, - lpad, rpad); + result = fill_padding(writer, len, + format->fill_char=='\0'?' ':format->fill_char, + lpad, rpad); + if (result == -1) + goto done; /* Then the source string. */ - if (PyUnicode_CopyCharacters(result, pos, value, 0, len) < 0) - Py_CLEAR(result); + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + value, 0, len); + writer->pos += (len + rpad); + result = 0; done: - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -780,11 +803,11 @@ typedef PyObject* (*IntOrLongToString)(PyObject *value, int base); -static PyObject * -format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, - IntOrLongToString tostring) +static int +format_long_internal(PyObject *value, const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; PyObject *tmp = NULL; Py_ssize_t inumeric_chars; @@ -798,7 +821,6 @@ Py_ssize_t prefix = 0; NumberFieldWidths spec; long x; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -872,13 +894,23 @@ break; } + if (format->sign != '+' && format->sign != ' ' + && format->width == -1 + && format->type != 'X' && format->type != 'n' + && !format->thousands_separators + && PyLong_CheckExact(value)) + { + /* Fast path */ + return _PyLong_FormatWriter(writer, value, base, format->alternate); + } + /* The number of prefix chars is the same as the leading chars to skip */ if (format->alternate) n_prefix = leading_chars_to_skip; /* Do the hard part, converting to a string in a given base */ - tmp = tostring(value, base); + tmp = _PyLong_Format(value, base); if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; @@ -914,23 +946,19 @@ &locale, format, &maxchar); /* Allocate the memory. */ - result = PyUnicode_New(n_total, maxchar); - if (!result) + if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) goto done; /* Populate the memory. */ - err = fill_number(result, 0, &spec, - tmp, inumeric_chars, inumeric_chars + n_digits, - tmp, prefix, - format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); - if (err) - Py_CLEAR(result); + result = fill_number(writer, &spec, + tmp, inumeric_chars, inumeric_chars + n_digits, + tmp, prefix, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, format->type == 'X'); done: Py_XDECREF(tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -938,16 +966,11 @@ /*********** float formatting *******************************************/ /************************************************************************/ -static PyObject* -strtounicode(char *charbuffer, Py_ssize_t len) -{ - return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, charbuffer, len); -} - /* much of this is taken from unicodeobject.c */ -static PyObject * +static int format_float_internal(PyObject *value, - const InternalFormatSpec *format) + const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { char *buf = NULL; /* buffer returned from PyOS_double_to_string */ Py_ssize_t n_digits; @@ -962,12 +985,11 @@ Py_ssize_t index; NumberFieldWidths spec; int flags = 0; - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; Py_UCS4 sign_char = '\0'; int float_type; /* Used to see if we have a nan, inf, or regular float. */ PyObject *unicode_tmp = NULL; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1024,13 +1046,25 @@ /* Since there is no unicode version of PyOS_double_to_string, just use the 8 bit version and then convert to unicode. */ - unicode_tmp = strtounicode(buf, n_digits); + unicode_tmp = _PyUnicode_FromASCII(buf, n_digits); + PyMem_Free(buf); if (unicode_tmp == NULL) goto done; - index = 0; + + if (format->sign != '+' && format->sign != ' ' + && format->width == -1 + && format->type != 'n' + && !format->thousands_separators) + { + /* Fast path */ + result = _PyUnicodeWriter_WriteStr(writer, unicode_tmp); + Py_DECREF(unicode_tmp); + return result; + } /* Is a sign character present in the output? If so, remember it and skip it */ + index = 0; if (PyUnicode_READ_CHAR(unicode_tmp, index) == '-') { sign_char = '-'; ++index; @@ -1055,24 +1089,19 @@ &locale, format, &maxchar); /* Allocate the memory. */ - result = PyUnicode_New(n_total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) goto done; /* Populate the memory. */ - err = fill_number(result, 0, &spec, - unicode_tmp, index, index + n_digits, - NULL, 0, - format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, 0); - if (err) - Py_CLEAR(result); + result = fill_number(writer, &spec, + unicode_tmp, index, index + n_digits, + NULL, 0, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, 0); done: - PyMem_Free(buf); Py_DECREF(unicode_tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -1080,9 +1109,10 @@ /*********** complex formatting *****************************************/ /************************************************************************/ -static PyObject * +static int format_complex_internal(PyObject *value, - const InternalFormatSpec *format) + const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { double re; double im; @@ -1106,11 +1136,10 @@ NumberFieldWidths re_spec; NumberFieldWidths im_spec; int flags = 0; - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; - int rkind; + enum PyUnicode_Kind rkind; void *rdata; - Py_ssize_t index; Py_UCS4 re_sign_char = '\0'; Py_UCS4 im_sign_char = '\0'; int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ @@ -1122,7 +1151,6 @@ Py_ssize_t total; PyObject *re_unicode_tmp = NULL; PyObject *im_unicode_tmp = NULL; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1191,12 +1219,12 @@ /* Since there is no unicode version of PyOS_double_to_string, just use the 8 bit version and then convert to unicode. */ - re_unicode_tmp = strtounicode(re_buf, n_re_digits); + re_unicode_tmp = _PyUnicode_FromASCII(re_buf, n_re_digits); if (re_unicode_tmp == NULL) goto done; i_re = 0; - im_unicode_tmp = strtounicode(im_buf, n_im_digits); + im_unicode_tmp = _PyUnicode_FromASCII(im_buf, n_im_digits); if (im_unicode_tmp == NULL) goto done; i_im = 0; @@ -1261,47 +1289,49 @@ if (lpad || rpad) maxchar = Py_MAX(maxchar, format->fill_char); - result = PyUnicode_New(total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, total, maxchar) == -1) goto done; - rkind = PyUnicode_KIND(result); - rdata = PyUnicode_DATA(result); + rkind = writer->kind; + rdata = writer->data; /* Populate the memory. First, the padding. */ - index = fill_padding(result, 0, - n_re_total + n_im_total + 1 + add_parens * 2, - format->fill_char=='\0' ? ' ' : format->fill_char, - lpad, rpad); + result = fill_padding(writer, + n_re_total + n_im_total + 1 + add_parens * 2, + format->fill_char=='\0' ? ' ' : format->fill_char, + lpad, rpad); + if (result == -1) + goto done; - if (add_parens) - PyUnicode_WRITE(rkind, rdata, index++, '('); + if (add_parens) { + PyUnicode_WRITE(rkind, rdata, writer->pos, '('); + writer->pos++; + } if (!skip_re) { - err = fill_number(result, index, &re_spec, - re_unicode_tmp, i_re, i_re + n_re_digits, - NULL, 0, - 0, - &locale, 0); - if (err) { - Py_CLEAR(result); + result = fill_number(writer, &re_spec, + re_unicode_tmp, i_re, i_re + n_re_digits, + NULL, 0, + 0, + &locale, 0); + if (result == -1) goto done; - } - index += n_re_total; } - err = fill_number(result, index, &im_spec, - im_unicode_tmp, i_im, i_im + n_im_digits, - NULL, 0, - 0, - &locale, 0); - if (err) { - Py_CLEAR(result); + result = fill_number(writer, &im_spec, + im_unicode_tmp, i_im, i_im + n_im_digits, + NULL, 0, + 0, + &locale, 0); + if (result == -1) goto done; + PyUnicode_WRITE(rkind, rdata, writer->pos, 'j'); + writer->pos++; + + if (add_parens) { + PyUnicode_WRITE(rkind, rdata, writer->pos, ')'); + writer->pos++; } - index += n_im_total; - PyUnicode_WRITE(rkind, rdata, index++, 'j'); - if (add_parens) - PyUnicode_WRITE(rkind, rdata, index++, ')'); + writer->pos += rpad; done: PyMem_Free(re_buf); @@ -1309,61 +1339,79 @@ Py_XDECREF(re_unicode_tmp); Py_XDECREF(im_unicode_tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } /************************************************************************/ /*********** built in formatters ****************************************/ /************************************************************************/ -PyObject * -_PyUnicode_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +format_obj(PyObject *obj, _PyUnicodeWriter *writer) +{ + PyObject *str; + int err; + + str = PyObject_Str(obj); + if (str == NULL) + return -1; + err = _PyUnicodeWriter_WriteStr(writer, str); + Py_DECREF(str); + return err; +} + +int +_PyUnicode_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { InternalFormatSpec format; - PyObject *result; + + assert(PyUnicode_Check(obj)); /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) - return PyObject_Str(obj); + if (start == end) { + if (PyUnicode_CheckExact(obj)) + return _PyUnicodeWriter_WriteStr(writer, obj); + else + return format_obj(obj, writer); + } /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, 's', '<')) - return NULL; + return -1; /* type conversion? */ switch (format.type) { case 's': /* no type conversion needed, already a string. do the formatting */ - result = format_string_internal(obj, &format); - if (result != NULL) - assert(_PyUnicode_CheckConsistency(result, 1)); - break; + return format_string_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - result = NULL; + return -1; } - return result; } -static PyObject* -format_int_or_long(PyObject* obj, PyObject* format_spec, - Py_ssize_t start, Py_ssize_t end, - IntOrLongToString tostring) +int +_PyLong_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - PyObject *result = NULL; - PyObject *tmp = NULL; + PyObject *tmp = NULL, *str = NULL; InternalFormatSpec format; + int result = -1; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ if (start == end) { - result = PyObject_Str(obj); - goto done; + if (PyLong_CheckExact(obj)) + return _PyLong_FormatWriter(writer, obj, 10, 0); + else + return format_obj(obj, writer); } /* parse the format_spec */ @@ -1382,7 +1430,7 @@ case 'n': /* no type conversion needed, already an int (or long). do the formatting */ - result = format_int_or_long_internal(obj, &format, tostring); + result = format_long_internal(obj, &format, writer); break; case 'e': @@ -1396,7 +1444,7 @@ tmp = PyNumber_Float(obj); if (tmp == NULL) goto done; - result = format_float_internal(tmp, &format); + result = format_float_internal(tmp, &format, writer); break; default: @@ -1407,41 +1455,27 @@ done: Py_XDECREF(tmp); + Py_XDECREF(str); return result; } -/* Need to define long_format as a function that will convert a long - to a string. In 3.0, _PyLong_Format has the correct signature. */ -#define long_format _PyLong_Format - -PyObject * -_PyLong_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +_PyFloat_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - return format_int_or_long(obj, format_spec, start, end, - long_format); -} - -PyObject * -_PyFloat_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) -{ - PyObject *result = NULL; InternalFormatSpec format; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) { - result = PyObject_Str(obj); - goto done; - } + if (start == end) + return format_obj(obj, writer); /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, '\0', '>')) - goto done; + return -1; /* type conversion? */ switch (format.type) { @@ -1455,38 +1489,32 @@ case 'n': case '%': /* no conversion, already a float. do the formatting */ - result = format_float_internal(obj, &format); - break; + return format_float_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; + return -1; } - -done: - return result; } -PyObject * -_PyComplex_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +_PyComplex_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - PyObject *result = NULL; InternalFormatSpec format; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) { - result = PyObject_Str(obj); - goto done; - } + if (start == end) + return format_obj(obj, writer); /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, '\0', '>')) - goto done; + return -1; /* type conversion? */ switch (format.type) { @@ -1499,15 +1527,11 @@ case 'G': case 'n': /* no conversion, already a complex. do the formatting */ - result = format_complex_internal(obj, &format); - break; + return format_complex_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; + return -1; } - -done: - return result; } diff --git a/Python/getargs.c b/Python/getargs.c --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1167,8 +1167,11 @@ case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - if (PyUnicode_Check(arg)) + if (PyUnicode_Check(arg)) { + if (PyUnicode_READY(arg) == -1) + RETURN_ERR_OCCURRED; *p = arg; + } else return converterr("str", arg, msgbuf, bufsize); break; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Added pyvenv docs to 'Python Setup and Usage' section.
by vinay.sajip May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/ace7c340d95d changeset: 77224:ace7c340d95d parent: 77219:01381723bc50 user: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> date: Tue May 29 12:52:14 2012 +0100 summary: Added pyvenv docs to 'Python Setup and Usage' section. files: Doc/library/venv.rst | 9 +-- Doc/using/index.rst | 2 +- Doc/using/scripts.rst | 66 +++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -54,7 +54,7 @@ The command, if run with ``-h``, will show the available options:: usage: pyvenv [-h] [--system-site-packages] [--symlink] [--clear] - ENV_DIR [ENV_DIR ...] + [--upgrade] ENV_DIR [ENV_DIR ...] Creates virtual Python environments in one or more target directories. @@ -69,12 +69,11 @@ --clear Delete the environment directory if it already exists. If not specified and the directory exists, an error is raised. - + --upgrade Upgrade the environment directory to use this version + of Python, assuming Python has been upgraded in-place. If the target directory already exists an error will be raised, unless -the ``--clear`` option was provided, in which case the target -directory will be deleted and virtual environment creation will -proceed as usual. +the ``--clear`` or ``--upgrade`` option was provided. The created ``pyvenv.cfg`` file also includes the ``include-system-site-packages`` key, set to ``true`` if ``venv`` is diff --git a/Doc/using/index.rst b/Doc/using/index.rst --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -17,4 +17,4 @@ unix.rst windows.rst mac.rst - + scripts.rst diff --git a/Doc/using/scripts.rst b/Doc/using/scripts.rst new file mode 100644 --- /dev/null +++ b/Doc/using/scripts.rst @@ -0,0 +1,66 @@ +.. _tools-and-scripts: + +Additional Tools and Scripts +============================ + +pyvenv - Creating virtual environments +-------------------------------------- + +Creation of virtual environments is done by executing the ``pyvenv`` +script:: + + pyvenv /path/to/new/virtual/environment + +Running this command creates the target directory (creating any parent +directories that don't exist already) and places a ``pyvenv.cfg`` file +in it with a ``home`` key pointing to the Python installation the +command was run from. It also creates a ``bin`` (or ``Scripts`` on +Windows) subdirectory containing a copy of the ``python`` binary (or +binaries, in the case of Windows) and the ``pysetup3`` script (to +facilitate easy installation of packages from PyPI into the new virtualenv). +It also creates an (initially empty) ``lib/pythonX.Y/site-packages`` +subdirectory (on Windows, this is ``Lib\site-packages``). + +.. highlight:: none + +On Windows, you may have to invoke the ``pyvenv`` script as follows, if you +don't have the relevant PATH and PATHEXT settings:: + + c:\Temp>c:\Python33\python c:\Python33\Tools\Scripts\pyvenv.py myenv + +or equivalently:: + + c:\Temp>c:\Python33\python -m venv myenv + +The command, if run with ``-h``, will show the available options:: + + usage: pyvenv [-h] [--system-site-packages] [--symlink] [--clear] + [--upgrade] ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + optional arguments: + -h, --help show this help message and exit + --system-site-packages Give access to the global site-packages dir to the + virtual environment. + --symlink Attempt to symlink rather than copy. + --clear Delete the environment directory if it already exists. + If not specified and the directory exists, an error is + raised. + --upgrade Upgrade the environment directory to use this version + of Python, assuming Python has been upgraded in-place. + +If the target directory already exists an error will be raised, unless +the ``--clear`` or ``--upgrade`` option was provided. + +The created ``pyvenv.cfg`` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Multiple paths can be given to ``pyvenv``, in which case an identical +virtualenv will be created, according to the given options, at each +provided path. + -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Issue #14744: Use the new _PyUnicodeWriter internal API to speed up str%args
by victor.stinner May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/22b56b0b8619 changeset: 77223:22b56b0b8619 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue May 29 12:57:52 2012 +0200 summary: Issue #14744: Use the new _PyUnicodeWriter internal API to speed up str%args and str.format(args) * Formatting string, int, float and complex use the _PyUnicodeWriter API. It avoids a temporary buffer in most cases. * Add _PyUnicodeWriter_WriteStr() to restore the PyAccu optimization: just keep a reference to the string if the output is only composed of one string * Disable overallocation when formatting the last argument of str%args and str.format(args) * Overallocation allocates at least 100 characters: add min_length attribute to the _PyUnicodeWriter structure * Add new private functions: _PyUnicode_FastCopyCharacters(), _PyUnicode_FastFill() and _PyUnicode_FromASCII() The speed up is around 20% in average. files: Include/complexobject.h | 10 +- Include/floatobject.h | 10 +- Include/longobject.h | 18 +- Include/unicodeobject.h | 95 +++- Misc/NEWS | 3 + Objects/complexobject.c | 17 +- Objects/floatobject.c | 27 +- Objects/longobject.c | 333 +++++++++---- Objects/stringlib/asciilib.h | 2 +- Objects/stringlib/unicode_format.h | 48 +- Objects/unicodeobject.c | 372 +++++++++++---- Python/formatter_unicode.c | 412 ++++++++-------- 12 files changed, 894 insertions(+), 453 deletions(-) diff --git a/Include/complexobject.h b/Include/complexobject.h --- a/Include/complexobject.h +++ b/Include/complexobject.h @@ -63,10 +63,12 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyComplex_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif #ifdef __cplusplus diff --git a/Include/floatobject.h b/Include/floatobject.h --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -112,10 +112,12 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Include/longobject.h b/Include/longobject.h --- a/Include/longobject.h +++ b/Include/longobject.h @@ -151,14 +151,22 @@ /* _PyLong_Format: Convert the long to a string object with given base, appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *aa, int base); +PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); + +PyAPI_FUNC(int) _PyLong_FormatWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + int base, + int alternate); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ /* These aren't really part of the long object, but they're handy. The diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -648,8 +648,20 @@ Py_ssize_t from_start, Py_ssize_t how_many ); + +/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so + may crash if parameters are invalid (e.g. if the output string + is too short). */ +PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( + PyObject *to, + Py_ssize_t to_start, + PyObject *from, + Py_ssize_t from_start, + Py_ssize_t how_many + ); #endif +#ifndef Py_LIMITED_API /* Fill a string with a character: write fill_char into unicode[start:start+length]. @@ -658,13 +670,21 @@ Return the number of written character, or return -1 and raise an exception on error. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( PyObject *unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char ); + +/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash + if parameters are invalid (e.g. if length is longer than the string). */ +PyAPI_FUNC(void) _PyUnicode_FastFill( + PyObject *unicode, + Py_ssize_t start, + Py_ssize_t length, + Py_UCS4 fill_char + ); #endif /* Create a Unicode Object from the Py_UNICODE buffer u of the given @@ -696,13 +716,19 @@ const char *u /* UTF-8 encoded string */ ); +#ifndef Py_LIMITED_API /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. Scan the string to find the maximum character. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( int kind, const void *buffer, Py_ssize_t size); + +/* Create a new string from a buffer of ASCII characters. + WARNING: Don't check if the string contains any non-ASCII character. */ +PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( + const char *buffer, + Py_ssize_t size); #endif PyAPI_FUNC(PyObject*) PyUnicode_Substring( @@ -865,12 +891,69 @@ ); #ifndef Py_LIMITED_API +typedef struct { + PyObject *buffer; + void *data; + enum PyUnicode_Kind kind; + Py_UCS4 maxchar; + Py_ssize_t size; + Py_ssize_t pos; + /* minimum length of the buffer when overallocation is enabled, + see _PyUnicodeWriter_Init() */ + Py_ssize_t min_length; + struct { + unsigned char overallocate:1; + /* If readonly is 1, buffer is a shared string (cannot be modified) + and size is set to 0. */ + unsigned char readonly:1; + } flags; +} _PyUnicodeWriter ; + +/* Initialize a Unicode writer. + + If min_length is greater than zero, _PyUnicodeWriter_Prepare() + overallocates the buffer and min_length is the minimum length in characters + of the buffer. */ +PyAPI_FUNC(void) +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length); + +/* Prepare the buffer to write 'length' characters + with the specified maximum character. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ + (((MAXCHAR) <= (WRITER)->maxchar \ + && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ + ? 0 \ + : (((LENGTH) == 0) \ + ? 0 \ + : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) + +/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro + instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar); + +PyAPI_FUNC(int) +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str); + +PyAPI_FUNC(PyObject *) +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); + +PyAPI_FUNC(void) +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); +#endif + +#ifndef Py_LIMITED_API /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ - Issue #14835: Make plistlib output empty arrays & dicts like OS X. Patch by Sidney San Martín. +- Issue #14744: Use the new _PyUnicodeWriter internal API to speed up + str%args and str.format(args). + - Issue #14930: Make memoryview objects weakrefable. - Issue #14775: Fix a potential quadratic dict build-up due to the garbage diff --git a/Objects/complexobject.c b/Objects/complexobject.c --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -699,11 +699,22 @@ complex__format__(PyObject* self, PyObject* args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) - return NULL; - return _PyComplex_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + return NULL; + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyComplex_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } #if 0 diff --git a/Objects/floatobject.c b/Objects/floatobject.c --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -267,13 +267,15 @@ float_repr(PyFloatObject *v) { PyObject *result; - char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), - 'r', 0, - Py_DTSF_ADD_DOT_0, - NULL); + char *buf; + + buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), + 'r', 0, + Py_DTSF_ADD_DOT_0, + NULL); if (!buf) return PyErr_NoMemory(); - result = PyUnicode_FromString(buf); + result = _PyUnicode_FromASCII(buf, strlen(buf)); PyMem_Free(buf); return result; } @@ -1703,11 +1705,22 @@ float__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyFloat_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(float__format__doc, diff --git a/Objects/longobject.c b/Objects/longobject.c --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1550,20 +1550,22 @@ string. (Return value is non-shared so that callers can modify the returned value if necessary.) */ -static PyObject * -long_to_decimal_string(PyObject *aa) +static int +long_to_decimal_string_internal(PyObject *aa, + PyObject **p_output, + _PyUnicodeWriter *writer) { PyLongObject *scratch, *a; PyObject *str; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; - unsigned char *p; int negative; + enum PyUnicode_Kind kind; a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); negative = Py_SIZE(a) < 0; @@ -1580,13 +1582,13 @@ if (size_a > PY_SSIZE_T_MAX / PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "long is too large to format"); - return NULL; + return -1; } /* the expression size_a * PyLong_SHIFT is now safe from overflow */ size = 1 + size_a * PyLong_SHIFT / (3 * _PyLong_DECIMAL_SHIFT); scratch = _PyLong_New(size); if (scratch == NULL) - return NULL; + return -1; /* convert array of base _PyLong_BASE digits in pin to an array of base _PyLong_DECIMAL_BASE digits in pout, following Knuth (TAOCP, @@ -1609,7 +1611,7 @@ /* check for keyboard interrupt */ SIGCHECK({ Py_DECREF(scratch); - return NULL; + return -1; }); } /* pout should have at least one digit, so that the case when a = 0 @@ -1625,65 +1627,113 @@ tenpow *= 10; strlen++; } - str = PyUnicode_New(strlen, '9'); - if (str == NULL) { - Py_DECREF(scratch); + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, strlen, '9') == -1) + return -1; + kind = writer->kind; + str = NULL; + } + else { + str = PyUnicode_New(strlen, '9'); + if (str == NULL) { + Py_DECREF(scratch); + return -1; + } + kind = PyUnicode_KIND(str); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ + else \ + p = (TYPE*)PyUnicode_DATA(str) + strlen; \ + \ + *p = '\0'; \ + /* pout[0] through pout[size-2] contribute exactly \ + _PyLong_DECIMAL_SHIFT digits each */ \ + for (i=0; i < size - 1; i++) { \ + rem = pout[i]; \ + for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } \ + } \ + /* pout[size-1]: always produce at least one decimal digit */ \ + rem = pout[i]; \ + do { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } while (rem != 0); \ + \ + /* and sign */ \ + if (negative) \ + *--p = '-'; \ + \ + /* check we've counted correctly */ \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(str)); \ + } while (0) + + /* fill the string right-to-left */ + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS + + Py_DECREF(scratch); + if (writer) { + writer->pos += strlen; + } + else { + assert(_PyUnicode_CheckConsistency(str, 1)); + *p_output = (PyObject *)str; + } + return 0; +} + +static PyObject * +long_to_decimal_string(PyObject *aa) +{ + PyObject *v; + if (long_to_decimal_string_internal(aa, &v, NULL) == -1) return NULL; - } - - /* fill the string right-to-left */ - assert(PyUnicode_KIND(str) == PyUnicode_1BYTE_KIND); - p = PyUnicode_1BYTE_DATA(str) + strlen; - *p = '\0'; - /* pout[0] through pout[size-2] contribute exactly - _PyLong_DECIMAL_SHIFT digits each */ - for (i=0; i < size - 1; i++) { - rem = pout[i]; - for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { - *--p = '0' + rem % 10; - rem /= 10; - } - } - /* pout[size-1]: always produce at least one decimal digit */ - rem = pout[i]; - do { - *--p = '0' + rem % 10; - rem /= 10; - } while (rem != 0); - - /* and sign */ - if (negative) - *--p = '-'; - - /* check we've counted correctly */ - assert(p == PyUnicode_1BYTE_DATA(str)); - assert(_PyUnicode_CheckConsistency(str, 1)); - Py_DECREF(scratch); - return (PyObject *)str; + return v; } /* Convert a long int object to a string, using a given conversion base, - which should be one of 2, 8, 10 or 16. Return a string object. - If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x'. */ - -PyObject * -_PyLong_Format(PyObject *aa, int base) + which should be one of 2, 8 or 16. Return a string object. + If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x' + if alternate is nonzero. */ + +static int +long_format_binary(PyObject *aa, int base, int alternate, + PyObject **p_output, _PyUnicodeWriter *writer) { register PyLongObject *a = (PyLongObject *)aa; PyObject *v; Py_ssize_t sz; Py_ssize_t size_a; - Py_UCS1 *p; + enum PyUnicode_Kind kind; int negative; int bits; - assert(base == 2 || base == 8 || base == 10 || base == 16); - if (base == 10) - return long_to_decimal_string((PyObject *)a); - + assert(base == 2 || base == 8 || base == 16); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); negative = Py_SIZE(a) < 0; @@ -1706,7 +1756,7 @@ /* Compute exact length 'sz' of output string. */ if (size_a == 0) { - sz = 3; + sz = 1; } else { Py_ssize_t size_a_in_bits; @@ -1714,56 +1764,126 @@ if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "int is too large to format"); - return NULL; + return -1; } size_a_in_bits = (size_a - 1) * PyLong_SHIFT + bits_in_digit(a->ob_digit[size_a - 1]); - /* Allow 2 characters for prefix and 1 for a '-' sign. */ - sz = 2 + negative + (size_a_in_bits + (bits - 1)) / bits; - } - - v = PyUnicode_New(sz, 'x'); - if (v == NULL) { + /* Allow 1 character for a '-' sign. */ + sz = negative + (size_a_in_bits + (bits - 1)) / bits; + } + if (alternate) { + /* 2 characters for prefix */ + sz += 2; + } + + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) + return -1; + kind = writer->kind; + v = NULL; + } + else { + v = PyUnicode_New(sz, 'x'); + if (v == NULL) + return -1; + kind = PyUnicode_KIND(v); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ + else \ + p = (TYPE*)PyUnicode_DATA(v) + sz; \ + \ + if (size_a == 0) { \ + *--p = '0'; \ + } \ + else { \ + /* JRH: special case for power-of-2 bases */ \ + twodigits accum = 0; \ + int accumbits = 0; /* # of bits in accum */ \ + Py_ssize_t i; \ + for (i = 0; i < size_a; ++i) { \ + accum |= (twodigits)a->ob_digit[i] << accumbits; \ + accumbits += PyLong_SHIFT; \ + assert(accumbits >= bits); \ + do { \ + char cdigit; \ + cdigit = (char)(accum & (base - 1)); \ + cdigit += (cdigit < 10) ? '0' : 'a'-10; \ + *--p = cdigit; \ + accumbits -= bits; \ + accum >>= bits; \ + } while (i < size_a-1 ? accumbits >= bits : accum > 0); \ + } \ + } \ + \ + if (alternate) { \ + if (base == 16) \ + *--p = 'x'; \ + else if (base == 8) \ + *--p = 'o'; \ + else /* (base == 2) */ \ + *--p = 'b'; \ + *--p = '0'; \ + } \ + if (negative) \ + *--p = '-'; \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(v)); \ + } while (0) + + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS + + if (writer) { + writer->pos += sz; + } + else { + assert(_PyUnicode_CheckConsistency(v, 1)); + *p_output = v; + } + return 0; +} + +PyObject * +_PyLong_Format(PyObject *obj, int base) +{ + PyObject *str; + int err; + if (base == 10) + err = long_to_decimal_string_internal(obj, &str, NULL); + else + err = long_format_binary(obj, base, 1, &str, NULL); + if (err == -1) return NULL; - } - assert(PyUnicode_KIND(v) == PyUnicode_1BYTE_KIND); - - p = PyUnicode_1BYTE_DATA(v) + sz; - if (size_a == 0) { - *--p = '0'; - } - else { - /* JRH: special case for power-of-2 bases */ - twodigits accum = 0; - int accumbits = 0; /* # of bits in accum */ - Py_ssize_t i; - for (i = 0; i < size_a; ++i) { - accum |= (twodigits)a->ob_digit[i] << accumbits; - accumbits += PyLong_SHIFT; - assert(accumbits >= bits); - do { - char cdigit; - cdigit = (char)(accum & (base - 1)); - cdigit += (cdigit < 10) ? '0' : 'a'-10; - *--p = cdigit; - accumbits -= bits; - accum >>= bits; - } while (i < size_a-1 ? accumbits >= bits : accum > 0); - } - } - - if (base == 16) - *--p = 'x'; - else if (base == 8) - *--p = 'o'; - else /* (base == 2) */ - *--p = 'b'; - *--p = '0'; - if (negative) - *--p = '-'; - assert(p == PyUnicode_1BYTE_DATA(v)); - assert(_PyUnicode_CheckConsistency(v, 1)); - return v; + return str; +} + +int +_PyLong_FormatWriter(_PyUnicodeWriter *writer, + PyObject *obj, + int base, int alternate) +{ + if (base == 10) + return long_to_decimal_string_internal(obj, NULL, writer); + else + return long_format_binary(obj, base, alternate, NULL, writer); } /* Table of digit values for 8-bit string -> integer conversion. @@ -4232,11 +4352,22 @@ long__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyLong_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyLong_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } /* Return a pair (q, r) such that a = b * q + r, and diff --git a/Objects/stringlib/asciilib.h b/Objects/stringlib/asciilib.h --- a/Objects/stringlib/asciilib.h +++ b/Objects/stringlib/asciilib.h @@ -18,7 +18,7 @@ #define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL #define STRINGLIB_STR PyUnicode_1BYTE_DATA #define STRINGLIB_LEN PyUnicode_GET_LENGTH -#define STRINGLIB_NEW unicode_fromascii +#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((char*)(STR),(LEN)) #define STRINGLIB_RESIZE not_supported #define STRINGLIB_CHECK PyUnicode_Check #define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -499,26 +499,26 @@ int ok = 0; PyObject *result = NULL; PyObject *format_spec_object = NULL; - PyObject *(*formatter)(PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; - Py_ssize_t len; + int (*formatter) (_PyUnicodeWriter*, PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; + int err; /* If we know the type exactly, skip the lookup of __format__ and just call the formatter directly. */ if (PyUnicode_CheckExact(fieldobj)) - formatter = _PyUnicode_FormatAdvanced; + formatter = _PyUnicode_FormatAdvancedWriter; else if (PyLong_CheckExact(fieldobj)) - formatter =_PyLong_FormatAdvanced; + formatter = _PyLong_FormatAdvancedWriter; else if (PyFloat_CheckExact(fieldobj)) - formatter = _PyFloat_FormatAdvanced; - - /* XXX: for 2.6, convert format_spec to the appropriate type - (unicode, str) */ + formatter = _PyFloat_FormatAdvancedWriter; + else if (PyComplex_CheckExact(fieldobj)) + formatter = _PyComplex_FormatAdvancedWriter; if (formatter) { /* we know exactly which formatter will be called when __format__ is looked up, so call it directly, instead. */ - result = formatter(fieldobj, format_spec->str, - format_spec->start, format_spec->end); + err = formatter(writer, fieldobj, format_spec->str, + format_spec->start, format_spec->end); + return (err == 0); } else { /* We need to create an object out of the pointers we have, because @@ -536,17 +536,11 @@ } if (result == NULL) goto done; - if (PyUnicode_READY(result) == -1) + + if (_PyUnicodeWriter_WriteStr(writer, result) == -1) goto done; + ok = 1; - len = PyUnicode_GET_LENGTH(result); - if (_PyUnicodeWriter_Prepare(writer, - len, PyUnicode_MAX_CHAR_VALUE(result)) == -1) - goto done; - copy_characters(writer->buffer, writer->pos, - result, 0, len); - writer->pos += len; - ok = 1; done: Py_XDECREF(format_spec_object); Py_XDECREF(result); @@ -897,16 +891,19 @@ err = _PyUnicodeWriter_Prepare(writer, sublen, maxchar); if (err == -1) return 0; - copy_characters(writer->buffer, writer->pos, - literal.str, literal.start, sublen); + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + literal.str, literal.start, sublen); writer->pos += sublen; } - if (field_present) + if (field_present) { + if (iter.str.start == iter.str.end) + writer->flags.overallocate = 0; if (!output_markup(&field_name, &format_spec, format_spec_needs_expanding, conversion, writer, args, kwargs, recursion_depth, auto_number)) return 0; + } } return result; } @@ -921,7 +918,7 @@ int recursion_depth, AutoNumber *auto_number) { _PyUnicodeWriter writer; - Py_ssize_t initlen; + Py_ssize_t minlen; /* check the recursion level */ if (recursion_depth <= 0) { @@ -930,9 +927,8 @@ return NULL; } - initlen = PyUnicode_GET_LENGTH(input->str) + 100; - if (_PyUnicodeWriter_Init(&writer, initlen, 127) == -1) - return NULL; + minlen = PyUnicode_GET_LENGTH(input->str) + 100; + _PyUnicodeWriter_Init(&writer, minlen); if (!do_markup(input, args, kwargs, &writer, recursion_depth, auto_number)) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -225,16 +225,10 @@ /* forward */ static PyUnicodeObject *_PyUnicode_New(Py_ssize_t length); static PyObject* get_latin1_char(unsigned char ch); -static void copy_characters( - PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many); static int unicode_modifiable(PyObject *unicode); static PyObject * -unicode_fromascii(const unsigned char *s, Py_ssize_t size); -static PyObject * _PyUnicode_FromUCS1(const unsigned char *s, Py_ssize_t size); static PyObject * _PyUnicode_FromUCS2(const Py_UCS2 *s, Py_ssize_t size); @@ -783,7 +777,7 @@ return NULL; copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode)); - copy_characters(copy, 0, unicode, 0, copy_length); + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length); return copy; } else { @@ -1154,15 +1148,16 @@ assert(0 <= from_start); assert(0 <= to_start); assert(PyUnicode_Check(from)); - assert(PyUnicode_Check(to)); assert(PyUnicode_IS_READY(from)); - assert(PyUnicode_IS_READY(to)); assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); - assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); if (how_many == 0) return 0; + assert(PyUnicode_Check(to)); + assert(PyUnicode_IS_READY(to)); + assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); + from_kind = PyUnicode_KIND(from); from_data = PyUnicode_DATA(from); to_kind = PyUnicode_KIND(to); @@ -1267,10 +1262,10 @@ return 0; } -static void -copy_characters(PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many) +void +_PyUnicode_FastCopyCharacters( + PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) { (void)_copy_characters(to, to_start, from, from_start, how_many, 0); } @@ -1292,6 +1287,14 @@ if (PyUnicode_READY(to) == -1) return -1; + if (from_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + if (to_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } how_many = Py_MIN(PyUnicode_GET_LENGTH(from), how_many); if (to_start + how_many > PyUnicode_GET_LENGTH(to)) { PyErr_Format(PyExc_SystemError, @@ -1641,7 +1644,7 @@ maxchar); if (result == NULL) return -1; - PyUnicode_CopyCharacters(result, 0, *p_unicode, 0, length); + _PyUnicode_FastCopyCharacters(result, 0, *p_unicode, 0, length); Py_DECREF(*p_unicode); *p_unicode = result; return 0; @@ -1841,9 +1844,10 @@ /* Internal function, doesn't check maximum character */ -static PyObject* -unicode_fromascii(const unsigned char* s, Py_ssize_t size) -{ +PyObject* +_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size) +{ + const unsigned char *s = (const unsigned char *)buffer; PyObject *unicode; if (size == 1) { #ifdef Py_DEBUG @@ -2085,7 +2089,7 @@ return; } copy = PyUnicode_New(len, max_char); - copy_characters(copy, 0, unicode, 0, len); + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len); Py_DECREF(unicode); *p_unicode = copy; } @@ -2753,7 +2757,7 @@ (void) va_arg(vargs, char *); size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -2767,7 +2771,7 @@ Py_ssize_t size; assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); size = PyUnicode_GET_LENGTH(obj); - copy_characters(string, i, obj, 0, size); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); i += size; break; } @@ -2779,13 +2783,13 @@ if (obj) { size = PyUnicode_GET_LENGTH(obj); assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); - copy_characters(string, i, obj, 0, size); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); i += size; } else { size = PyUnicode_GET_LENGTH(*callresult); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; Py_DECREF(*callresult); } @@ -2800,7 +2804,7 @@ /* unused, since we already have the result */ (void) va_arg(vargs, PyObject *); assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); - copy_characters(string, i, *callresult, 0, size); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); @@ -4171,7 +4175,7 @@ if (unicode_widen(output, *outpos, PyUnicode_MAX_CHAR_VALUE(repunicode)) < 0) goto onError; - copy_characters(*output, *outpos, repunicode, 0, replen); + _PyUnicode_FastCopyCharacters(*output, *outpos, repunicode, 0, replen); *outpos += replen; } else { @@ -9216,12 +9220,14 @@ /* If the maxchar increased so that the kind changed, not all characters are representable anymore and we need to fix the string again. This only happens in very few cases. */ - copy_characters(v, 0, self, 0, PyUnicode_GET_LENGTH(self)); + _PyUnicode_FastCopyCharacters(v, 0, + self, 0, PyUnicode_GET_LENGTH(self)); maxchar_old = fixfct(v); assert(maxchar_old > 0 && maxchar_old <= maxchar_new); } else { - copy_characters(v, 0, u, 0, PyUnicode_GET_LENGTH(self)); + _PyUnicode_FastCopyCharacters(v, 0, + u, 0, PyUnicode_GET_LENGTH(self)); } Py_DECREF(u); assert(_PyUnicode_CheckConsistency(v, 1)); @@ -9603,7 +9609,7 @@ res_data += kind * seplen; } else { - copy_characters(res, res_offset, sep, 0, seplen); + _PyUnicode_FastCopyCharacters(res, res_offset, sep, 0, seplen); res_offset += seplen; } } @@ -9616,7 +9622,7 @@ res_data += kind * itemlen; } else { - copy_characters(res, res_offset, item, 0, itemlen); + _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); res_offset += itemlen; } } @@ -9663,13 +9669,25 @@ } \ } while (0) +void +_PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, + Py_UCS4 fill_char) +{ + const enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + assert(PyUnicode_IS_READY(unicode)); + assert(unicode_modifiable(unicode)); + assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); + assert(start >= 0); + assert(start + length <= PyUnicode_GET_LENGTH(unicode)); + FILL(kind, data, fill_char, start, length); +} + Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char) { Py_ssize_t maxlen; - enum PyUnicode_Kind kind; - void *data; if (!PyUnicode_Check(unicode)) { PyErr_BadInternalCall(); @@ -9680,6 +9698,10 @@ if (unicode_check_modifiable(unicode)) return -1; + if (start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } if (fill_char > PyUnicode_MAX_CHAR_VALUE(unicode)) { PyErr_SetString(PyExc_ValueError, "fill character is bigger than " @@ -9692,9 +9714,7 @@ if (length <= 0) return 0; - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - FILL(kind, data, fill_char, start, length); + _PyUnicode_FastFill(unicode, start, length, fill_char); return length; } @@ -9734,7 +9754,7 @@ FILL(kind, data, fill, 0, left); if (right) FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); - copy_characters(u, left, self, 0, _PyUnicode_LENGTH(self)); + _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); assert(_PyUnicode_CheckConsistency(u, 1)); return u; } @@ -10058,7 +10078,7 @@ u = PyUnicode_New(slen, maxchar); if (!u) goto error; - copy_characters(u, 0, self, 0, slen); + _PyUnicode_FastCopyCharacters(u, 0, self, 0, slen); rkind = PyUnicode_KIND(u); PyUnicode_WRITE(rkind, PyUnicode_DATA(u), pos, u2); @@ -10626,8 +10646,8 @@ w = PyUnicode_New(new_len, maxchar); if (w == NULL) goto onError; - copy_characters(w, 0, u, 0, u_len); - copy_characters(w, u_len, v, 0, v_len); + _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); + _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); Py_DECREF(u); Py_DECREF(v); assert(_PyUnicode_CheckConsistency(w, 1)); @@ -10702,7 +10722,7 @@ goto error; } /* copy 'right' into the newly allocated area of 'left' */ - copy_characters(*p_left, left_len, right, 0, right_len); + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); } else { maxchar = PyUnicode_MAX_CHAR_VALUE(left); @@ -10713,8 +10733,8 @@ res = PyUnicode_New(new_len, maxchar); if (res == NULL) goto error; - copy_characters(res, 0, left, 0, left_len); - copy_characters(res, left_len, right, 0, right_len); + _PyUnicode_FastCopyCharacters(res, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(res, left_len, right, 0, right_len); Py_DECREF(left); *p_left = res; } @@ -11650,7 +11670,7 @@ length = end - start; if (PyUnicode_IS_ASCII(self)) { data = PyUnicode_1BYTE_DATA(self); - return unicode_fromascii(data + start, length); + return _PyUnicode_FromASCII((char*)(data + start), length); } else { kind = PyUnicode_KIND(self); @@ -12769,60 +12789,74 @@ return PyBool_FromLong(result); } -typedef struct { - PyObject *buffer; - void *data; - enum PyUnicode_Kind kind; - Py_UCS4 maxchar; - Py_ssize_t pos; -} _PyUnicodeWriter ; - Py_LOCAL_INLINE(void) _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { + writer->size = PyUnicode_GET_LENGTH(writer->buffer); writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); writer->data = PyUnicode_DATA(writer->buffer); writer->kind = PyUnicode_KIND(writer->buffer); } -Py_LOCAL(int) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) -{ - writer->pos = 0; - writer->buffer = PyUnicode_New(length, maxchar); - if (writer->buffer == NULL) - return -1; - _PyUnicodeWriter_Update(writer); - return 0; -} - -Py_LOCAL_INLINE(int) -_PyUnicodeWriter_Prepare(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) +void +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length) +{ + memset(writer, 0, sizeof(*writer)); +#ifdef Py_DEBUG + writer->kind = 5; /* invalid kind */ +#endif + writer->min_length = Py_MAX(min_length, 100); + writer->flags.overallocate = (min_length > 0); +} + +int +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar) { Py_ssize_t newlen; PyObject *newbuffer; + assert(length > 0); + if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); return -1; } newlen = writer->pos + length; - if (newlen > PyUnicode_GET_LENGTH(writer->buffer)) { - /* overallocate 25% to limit the number of resize */ - if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) - newlen += newlen / 4; - - if (maxchar > writer->maxchar) { + if (writer->buffer == NULL) { + if (writer->flags.overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + writer->buffer = PyUnicode_New(newlen, maxchar); + if (writer->buffer == NULL) + return -1; + _PyUnicodeWriter_Update(writer); + return 0; + } + + if (newlen > writer->size) { + if (writer->flags.overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + + if (maxchar > writer->maxchar || writer->flags.readonly) { /* resize + widen */ newbuffer = PyUnicode_New(newlen, maxchar); if (newbuffer == NULL) return -1; - PyUnicode_CopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); Py_DECREF(writer->buffer); + writer->flags.readonly = 0; } else { newbuffer = resize_compact(writer->buffer, newlen); @@ -12833,25 +12867,76 @@ _PyUnicodeWriter_Update(writer); } else if (maxchar > writer->maxchar) { - if (unicode_widen(&writer->buffer, writer->pos, maxchar) < 0) + assert(!writer->flags.readonly); + newbuffer = PyUnicode_New(writer->size, maxchar); + if (newbuffer == NULL) return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_DECREF(writer->buffer); + writer->buffer = newbuffer; _PyUnicodeWriter_Update(writer); } return 0; } -Py_LOCAL(PyObject *) +int +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) +{ + Py_UCS4 maxchar; + Py_ssize_t len; + + if (PyUnicode_READY(str) == -1) + return -1; + len = PyUnicode_GET_LENGTH(str); + if (len == 0) + return 0; + maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { + if (writer->buffer == NULL && !writer->flags.overallocate) { + Py_INCREF(str); + writer->buffer = str; + _PyUnicodeWriter_Update(writer); + writer->flags.readonly = 1; + writer->size = 0; + writer->pos += len; + return 0; + } + if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) + return -1; + } + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, 0, len); + writer->pos += len; + return 0; +} + +PyObject * _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) { - if (PyUnicode_Resize(&writer->buffer, writer->pos) < 0) { - Py_DECREF(writer->buffer); - return NULL; + if (writer->pos == 0) { + Py_XDECREF(writer->buffer); + Py_INCREF(unicode_empty); + return unicode_empty; + } + if (writer->flags.readonly) { + assert(PyUnicode_GET_LENGTH(writer->buffer) == writer->pos); + return writer->buffer; + } + if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { + PyObject *newbuffer; + newbuffer = resize_compact(writer->buffer, writer->pos); + if (newbuffer == NULL) { + Py_DECREF(writer->buffer); + return NULL; + } + writer->buffer = newbuffer; } assert(_PyUnicode_CheckConsistency(writer->buffer, 1)); return writer->buffer; } -Py_LOCAL(void) +void _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) { Py_CLEAR(writer->buffer); @@ -12874,14 +12959,24 @@ static PyObject * unicode__format__(PyObject* self, PyObject* args) { - PyObject *format_spec, *out; + PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - out = _PyUnicode_FormatAdvanced(self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); - return out; + if (PyUnicode_READY(self) == -1) + return NULL; + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyUnicode_FormatAdvancedWriter(&writer, + self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(p_format__doc__, @@ -13111,16 +13206,17 @@ /* Returns a new reference to a PyUnicode object, or NULL on failure. */ -static PyObject * -formatfloat(PyObject *v, int flags, int prec, int type) +static int +formatfloat(PyObject *v, int flags, int prec, int type, + PyObject **p_output, _PyUnicodeWriter *writer) { char *p; - PyObject *result; double x; + Py_ssize_t len; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) - return NULL; + return -1; if (prec < 0) prec = 6; @@ -13128,10 +13224,20 @@ p = PyOS_double_to_string(x, type, prec, (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL); if (p == NULL) - return NULL; - result = unicode_fromascii((unsigned char*)p, strlen(p)); + return -1; + len = strlen(p); + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) + return -1; + memcpy(writer->data + writer->pos * writer->kind, + p, + len); + writer->pos += len; + } + else + *p_output = _PyUnicode_FromASCII(p, len); PyMem_Free(p); - return result; + return 0; } /* formatlong() emulates the format codes d, u, o, x and X, and @@ -13267,7 +13373,7 @@ } if (!PyUnicode_Check(result) || len != PyUnicode_GET_LENGTH(result)) { PyObject *unicode; - unicode = unicode_fromascii((unsigned char *)buf, len); + unicode = _PyUnicode_FromASCII(buf, len); Py_DECREF(result); result = unicode; } @@ -13336,8 +13442,7 @@ fmtcnt = PyUnicode_GET_LENGTH(uformat); fmtpos = 0; - if (_PyUnicodeWriter_Init(&writer, fmtcnt + 100, 127) < 0) - goto onError; + _PyUnicodeWriter_Init(&writer, fmtcnt + 100); if (PyTuple_Check(args)) { arglen = PyTuple_Size(args); @@ -13368,8 +13473,8 @@ if (_PyUnicodeWriter_Prepare(&writer, sublen, maxchar) == -1) goto onError; - copy_characters(writer.buffer, writer.pos, - uformat, nonfmtpos, sublen); + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + uformat, nonfmtpos, sublen); writer.pos += sublen; } else { @@ -13530,6 +13635,8 @@ "incomplete format"); goto onError; } + if (fmtcnt == 0) + writer.flags.overallocate = 0; if (c == '%') { if (_PyUnicodeWriter_Prepare(&writer, 1, '%') == -1) @@ -13539,7 +13646,6 @@ continue; } - v = getnextarg(args, arglen, &argidx); if (v == NULL) goto onError; @@ -13552,6 +13658,13 @@ case 's': case 'r': case 'a': + if (PyLong_CheckExact(v) && width == -1 && prec == -1) { + /* Fast path */ + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + } + if (PyUnicode_CheckExact(v) && c == 's') { temp = v; Py_INCREF(temp); @@ -13572,6 +13685,32 @@ case 'o': case 'x': case 'X': + if (PyLong_CheckExact(v) + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + switch(c) + { + case 'd': + case 'i': + case 'u': + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'x': + if (_PyLong_FormatWriter(&writer, v, 16, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'o': + if (_PyLong_FormatWriter(&writer, v, 8, flags & F_ALT) == -1) + goto onError; + goto nextarg; + default: + break; + } + } + isnumok = 0; if (PyNumber_Check(v)) { PyObject *iobj=NULL; @@ -13611,10 +13750,20 @@ case 'F': case 'g': case 'G': + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (formatfloat(v, flags, prec, c, NULL, &writer) == -1) + goto onError; + goto nextarg; + } + sign = 1; if (flags & F_ZERO) fill = '0'; - temp = formatfloat(v, flags, prec, c); + if (formatfloat(v, flags, prec, c, &temp, NULL) == -1) + temp = NULL; break; case 'c': @@ -13622,6 +13771,14 @@ Py_UCS4 ch = formatchar(v); if (ch == (Py_UCS4) -1) goto onError; + if (width == -1 && prec == -1) { + /* Fast path */ + if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1) + goto onError; + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch); + writer.pos += 1; + goto nextarg; + } temp = PyUnicode_FromOrdinal(ch); break; } @@ -13638,6 +13795,16 @@ if (temp == NULL) goto onError; assert (PyUnicode_Check(temp)); + + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (_PyUnicodeWriter_WriteStr(&writer, temp) == -1) + goto onError; + goto nextarg; + } + if (PyUnicode_READY(temp) == -1) { Py_CLEAR(temp); goto onError; @@ -13676,15 +13843,15 @@ if (!(flags & F_LJUST)) { if (sign) { if ((width-1) > len) - bufmaxchar = Py_MAX(bufmaxchar, fill); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } else { if (width > len) - bufmaxchar = Py_MAX(bufmaxchar, fill); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } } maxchar = _PyUnicode_FindMaxChar(temp, 0, pindex+len); - bufmaxchar = Py_MAX(bufmaxchar, maxchar); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar); buflen = width; if (sign && len == width) @@ -13737,8 +13904,8 @@ } } - copy_characters(writer.buffer, writer.pos, - temp, pindex, len); + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + temp, pindex, len); writer.pos += len; if (width > len) { sublen = width - len; @@ -13746,6 +13913,7 @@ writer.pos += sublen; } +nextarg: if (dict && (argidx < arglen) && c != '%') { PyErr_SetString(PyExc_TypeError, "not all arguments converted during string formatting"); diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -316,21 +316,28 @@ /* Do the padding, and return a pointer to where the caller-supplied content goes. */ static Py_ssize_t -fill_padding(PyObject *s, Py_ssize_t start, Py_ssize_t nchars, +fill_padding(_PyUnicodeWriter *writer, + Py_ssize_t nchars, Py_UCS4 fill_char, Py_ssize_t n_lpadding, Py_ssize_t n_rpadding) { + Py_ssize_t pos; + /* Pad on left. */ - if (n_lpadding) - PyUnicode_Fill(s, start, start + n_lpadding, fill_char); + if (n_lpadding) { + pos = writer->pos; + _PyUnicode_FastFill(writer->buffer, pos, n_lpadding, fill_char); + } /* Pad on right. */ - if (n_rpadding) - PyUnicode_Fill(s, start + nchars + n_lpadding, - start + nchars + n_lpadding + n_rpadding, fill_char); + if (n_rpadding) { + pos = writer->pos + nchars + n_lpadding; + _PyUnicode_FastFill(writer->buffer, pos, n_rpadding, fill_char); + } /* Pointer to the user content. */ - return start + n_lpadding; + writer->pos += n_lpadding; + return 0; } /************************************************************************/ @@ -541,7 +548,7 @@ as determined in calc_number_widths(). Return -1 on error, or 0 on success. */ static int -fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec, +fill_number(_PyUnicodeWriter *writer, const NumberFieldWidths *spec, PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end, PyObject *prefix, Py_ssize_t p_start, Py_UCS4 fill_char, @@ -549,36 +556,38 @@ { /* Used to keep track of digits, decimal, and remainder. */ Py_ssize_t d_pos = d_start; - unsigned int kind = PyUnicode_KIND(out); - void *data = PyUnicode_DATA(out); + const enum PyUnicode_Kind kind = writer->kind; + const void *data = writer->data; Py_ssize_t r; if (spec->n_lpadding) { - PyUnicode_Fill(out, pos, pos + spec->n_lpadding, fill_char); - pos += spec->n_lpadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_lpadding, fill_char); + writer->pos += spec->n_lpadding; } if (spec->n_sign == 1) { - PyUnicode_WRITE(kind, data, pos++, spec->sign); + PyUnicode_WRITE(kind, data, writer->pos, spec->sign); + writer->pos++; } if (spec->n_prefix) { - if (PyUnicode_CopyCharacters(out, pos, - prefix, p_start, - spec->n_prefix) < 0) - return -1; + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + prefix, p_start, + spec->n_prefix); if (toupper) { Py_ssize_t t; for (t = 0; t < spec->n_prefix; t++) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + Py_UCS4 c = PyUnicode_READ(kind, data, writer->pos + t); c = Py_TOUPPER(c); assert (c <= 127); - PyUnicode_WRITE(kind, data, pos + t, c); + PyUnicode_WRITE(kind, data, writer->pos + t, c); } } - pos += spec->n_prefix; + writer->pos += spec->n_prefix; } if (spec->n_spadding) { - PyUnicode_Fill(out, pos, pos + spec->n_spadding, fill_char); - pos += spec->n_spadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_spadding, fill_char); + writer->pos += spec->n_spadding; } /* Only for type 'c' special case, it has no digits. */ @@ -594,7 +603,7 @@ return -1; } r = _PyUnicode_InsertThousandsGrouping( - out, pos, + writer->buffer, writer->pos, spec->n_grouped_digits, pdigits + kind * d_pos, spec->n_digits, spec->n_min_width, @@ -609,34 +618,38 @@ if (toupper) { Py_ssize_t t; for (t = 0; t < spec->n_grouped_digits; t++) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos + t); + Py_UCS4 c = PyUnicode_READ(kind, data, writer->pos + t); c = Py_TOUPPER(c); if (c > 127) { PyErr_SetString(PyExc_SystemError, "non-ascii grouped digit"); return -1; } - PyUnicode_WRITE(kind, data, pos + t, c); + PyUnicode_WRITE(kind, data, writer->pos + t, c); } } - pos += spec->n_grouped_digits; + writer->pos += spec->n_grouped_digits; if (spec->n_decimal) { - if (PyUnicode_CopyCharacters(out, pos, locale->decimal_point, 0, spec->n_decimal) < 0) - return -1; - pos += spec->n_decimal; + _PyUnicode_FastCopyCharacters( + writer->buffer, writer->pos, + locale->decimal_point, 0, spec->n_decimal); + writer->pos += spec->n_decimal; d_pos += 1; } if (spec->n_remainder) { - if (PyUnicode_CopyCharacters(out, pos, digits, d_pos, spec->n_remainder) < 0) - return -1; - pos += spec->n_remainder; + _PyUnicode_FastCopyCharacters( + writer->buffer, writer->pos, + digits, d_pos, spec->n_remainder); + writer->pos += spec->n_remainder; d_pos += spec->n_remainder; } if (spec->n_rpadding) { - PyUnicode_Fill(out, pos, pos + spec->n_rpadding, fill_char); - pos += spec->n_rpadding; + _PyUnicode_FastFill(writer->buffer, + writer->pos, spec->n_rpadding, + fill_char); + writer->pos += spec->n_rpadding; } return 0; } @@ -707,17 +720,20 @@ /*********** string formatting ******************************************/ /************************************************************************/ -static PyObject * -format_string_internal(PyObject *value, const InternalFormatSpec *format) +static int +format_string_internal(PyObject *value, const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { Py_ssize_t lpad; Py_ssize_t rpad; Py_ssize_t total; - Py_ssize_t pos; - Py_ssize_t len = PyUnicode_GET_LENGTH(value); - PyObject *result = NULL; + Py_ssize_t len; + int result = -1; Py_UCS4 maxchar; + assert(PyUnicode_IS_READY(value)); + len = PyUnicode_GET_LENGTH(value); + /* sign is not allowed on strings */ if (format->sign != '\0') { PyErr_SetString(PyExc_ValueError, @@ -741,6 +757,11 @@ goto done; } + if (format->width == -1 && format->precision == -1) { + /* Fast path */ + return _PyUnicodeWriter_WriteStr(writer, value); + } + /* if precision is specified, output no more that format.precision characters */ if (format->precision >= 0 && len >= format->precision) { @@ -754,21 +775,23 @@ maxchar = Py_MAX(maxchar, format->fill_char); /* allocate the resulting string */ - result = PyUnicode_New(total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, total, maxchar) == -1) goto done; /* Write into that space. First the padding. */ - pos = fill_padding(result, 0, len, - format->fill_char=='\0'?' ':format->fill_char, - lpad, rpad); + result = fill_padding(writer, len, + format->fill_char=='\0'?' ':format->fill_char, + lpad, rpad); + if (result == -1) + goto done; /* Then the source string. */ - if (PyUnicode_CopyCharacters(result, pos, value, 0, len) < 0) - Py_CLEAR(result); + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + value, 0, len); + writer->pos += (len + rpad); + result = 0; done: - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -780,11 +803,11 @@ typedef PyObject* (*IntOrLongToString)(PyObject *value, int base); -static PyObject * -format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, - IntOrLongToString tostring) +static int +format_long_internal(PyObject *value, const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; PyObject *tmp = NULL; Py_ssize_t inumeric_chars; @@ -798,7 +821,6 @@ Py_ssize_t prefix = 0; NumberFieldWidths spec; long x; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -872,13 +894,23 @@ break; } + if (format->sign != '+' && format->sign != ' ' + && format->width == -1 + && format->type != 'X' && format->type != 'n' + && !format->thousands_separators + && PyLong_CheckExact(value)) + { + /* Fast path */ + return _PyLong_FormatWriter(writer, value, base, format->alternate); + } + /* The number of prefix chars is the same as the leading chars to skip */ if (format->alternate) n_prefix = leading_chars_to_skip; /* Do the hard part, converting to a string in a given base */ - tmp = tostring(value, base); + tmp = _PyLong_Format(value, base); if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; @@ -914,23 +946,19 @@ &locale, format, &maxchar); /* Allocate the memory. */ - result = PyUnicode_New(n_total, maxchar); - if (!result) + if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) goto done; /* Populate the memory. */ - err = fill_number(result, 0, &spec, - tmp, inumeric_chars, inumeric_chars + n_digits, - tmp, prefix, - format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); - if (err) - Py_CLEAR(result); + result = fill_number(writer, &spec, + tmp, inumeric_chars, inumeric_chars + n_digits, + tmp, prefix, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, format->type == 'X'); done: Py_XDECREF(tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -938,16 +966,11 @@ /*********** float formatting *******************************************/ /************************************************************************/ -static PyObject* -strtounicode(char *charbuffer, Py_ssize_t len) -{ - return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, charbuffer, len); -} - /* much of this is taken from unicodeobject.c */ -static PyObject * +static int format_float_internal(PyObject *value, - const InternalFormatSpec *format) + const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { char *buf = NULL; /* buffer returned from PyOS_double_to_string */ Py_ssize_t n_digits; @@ -962,12 +985,11 @@ Py_ssize_t index; NumberFieldWidths spec; int flags = 0; - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; Py_UCS4 sign_char = '\0'; int float_type; /* Used to see if we have a nan, inf, or regular float. */ PyObject *unicode_tmp = NULL; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1024,13 +1046,25 @@ /* Since there is no unicode version of PyOS_double_to_string, just use the 8 bit version and then convert to unicode. */ - unicode_tmp = strtounicode(buf, n_digits); + unicode_tmp = _PyUnicode_FromASCII(buf, n_digits); + PyMem_Free(buf); if (unicode_tmp == NULL) goto done; - index = 0; + + if (format->sign != '+' && format->sign != ' ' + && format->width == -1 + && format->type != 'n' + && !format->thousands_separators) + { + /* Fast path */ + result = _PyUnicodeWriter_WriteStr(writer, unicode_tmp); + Py_DECREF(unicode_tmp); + return result; + } /* Is a sign character present in the output? If so, remember it and skip it */ + index = 0; if (PyUnicode_READ_CHAR(unicode_tmp, index) == '-') { sign_char = '-'; ++index; @@ -1055,24 +1089,19 @@ &locale, format, &maxchar); /* Allocate the memory. */ - result = PyUnicode_New(n_total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, n_total, maxchar) == -1) goto done; /* Populate the memory. */ - err = fill_number(result, 0, &spec, - unicode_tmp, index, index + n_digits, - NULL, 0, - format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, 0); - if (err) - Py_CLEAR(result); + result = fill_number(writer, &spec, + unicode_tmp, index, index + n_digits, + NULL, 0, + format->fill_char == '\0' ? ' ' : format->fill_char, + &locale, 0); done: - PyMem_Free(buf); Py_DECREF(unicode_tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } @@ -1080,9 +1109,10 @@ /*********** complex formatting *****************************************/ /************************************************************************/ -static PyObject * +static int format_complex_internal(PyObject *value, - const InternalFormatSpec *format) + const InternalFormatSpec *format, + _PyUnicodeWriter *writer) { double re; double im; @@ -1106,11 +1136,10 @@ NumberFieldWidths re_spec; NumberFieldWidths im_spec; int flags = 0; - PyObject *result = NULL; + int result = -1; Py_UCS4 maxchar = 127; - int rkind; + enum PyUnicode_Kind rkind; void *rdata; - Py_ssize_t index; Py_UCS4 re_sign_char = '\0'; Py_UCS4 im_sign_char = '\0'; int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ @@ -1122,7 +1151,6 @@ Py_ssize_t total; PyObject *re_unicode_tmp = NULL; PyObject *im_unicode_tmp = NULL; - int err; /* Locale settings, either from the actual locale or from a hard-code pseudo-locale */ @@ -1191,12 +1219,12 @@ /* Since there is no unicode version of PyOS_double_to_string, just use the 8 bit version and then convert to unicode. */ - re_unicode_tmp = strtounicode(re_buf, n_re_digits); + re_unicode_tmp = _PyUnicode_FromASCII(re_buf, n_re_digits); if (re_unicode_tmp == NULL) goto done; i_re = 0; - im_unicode_tmp = strtounicode(im_buf, n_im_digits); + im_unicode_tmp = _PyUnicode_FromASCII(im_buf, n_im_digits); if (im_unicode_tmp == NULL) goto done; i_im = 0; @@ -1261,47 +1289,49 @@ if (lpad || rpad) maxchar = Py_MAX(maxchar, format->fill_char); - result = PyUnicode_New(total, maxchar); - if (result == NULL) + if (_PyUnicodeWriter_Prepare(writer, total, maxchar) == -1) goto done; - rkind = PyUnicode_KIND(result); - rdata = PyUnicode_DATA(result); + rkind = writer->kind; + rdata = writer->data; /* Populate the memory. First, the padding. */ - index = fill_padding(result, 0, - n_re_total + n_im_total + 1 + add_parens * 2, - format->fill_char=='\0' ? ' ' : format->fill_char, - lpad, rpad); + result = fill_padding(writer, + n_re_total + n_im_total + 1 + add_parens * 2, + format->fill_char=='\0' ? ' ' : format->fill_char, + lpad, rpad); + if (result == -1) + goto done; - if (add_parens) - PyUnicode_WRITE(rkind, rdata, index++, '('); + if (add_parens) { + PyUnicode_WRITE(rkind, rdata, writer->pos, '('); + writer->pos++; + } if (!skip_re) { - err = fill_number(result, index, &re_spec, - re_unicode_tmp, i_re, i_re + n_re_digits, - NULL, 0, - 0, - &locale, 0); - if (err) { - Py_CLEAR(result); + result = fill_number(writer, &re_spec, + re_unicode_tmp, i_re, i_re + n_re_digits, + NULL, 0, + 0, + &locale, 0); + if (result == -1) goto done; - } - index += n_re_total; } - err = fill_number(result, index, &im_spec, - im_unicode_tmp, i_im, i_im + n_im_digits, - NULL, 0, - 0, - &locale, 0); - if (err) { - Py_CLEAR(result); + result = fill_number(writer, &im_spec, + im_unicode_tmp, i_im, i_im + n_im_digits, + NULL, 0, + 0, + &locale, 0); + if (result == -1) goto done; + PyUnicode_WRITE(rkind, rdata, writer->pos, 'j'); + writer->pos++; + + if (add_parens) { + PyUnicode_WRITE(rkind, rdata, writer->pos, ')'); + writer->pos++; } - index += n_im_total; - PyUnicode_WRITE(rkind, rdata, index++, 'j'); - if (add_parens) - PyUnicode_WRITE(rkind, rdata, index++, ')'); + writer->pos += rpad; done: PyMem_Free(re_buf); @@ -1309,61 +1339,79 @@ Py_XDECREF(re_unicode_tmp); Py_XDECREF(im_unicode_tmp); free_locale_info(&locale); - assert(!result || _PyUnicode_CheckConsistency(result, 1)); return result; } /************************************************************************/ /*********** built in formatters ****************************************/ /************************************************************************/ -PyObject * -_PyUnicode_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +format_obj(PyObject *obj, _PyUnicodeWriter *writer) +{ + PyObject *str; + int err; + + str = PyObject_Str(obj); + if (str == NULL) + return -1; + err = _PyUnicodeWriter_WriteStr(writer, str); + Py_DECREF(str); + return err; +} + +int +_PyUnicode_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { InternalFormatSpec format; - PyObject *result; + + assert(PyUnicode_Check(obj)); /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) - return PyObject_Str(obj); + if (start == end) { + if (PyUnicode_CheckExact(obj)) + return _PyUnicodeWriter_WriteStr(writer, obj); + else + return format_obj(obj, writer); + } /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, 's', '<')) - return NULL; + return -1; /* type conversion? */ switch (format.type) { case 's': /* no type conversion needed, already a string. do the formatting */ - result = format_string_internal(obj, &format); - if (result != NULL) - assert(_PyUnicode_CheckConsistency(result, 1)); - break; + return format_string_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - result = NULL; + return -1; } - return result; } -static PyObject* -format_int_or_long(PyObject* obj, PyObject* format_spec, - Py_ssize_t start, Py_ssize_t end, - IntOrLongToString tostring) +int +_PyLong_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - PyObject *result = NULL; - PyObject *tmp = NULL; + PyObject *tmp = NULL, *str = NULL; InternalFormatSpec format; + int result = -1; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ if (start == end) { - result = PyObject_Str(obj); - goto done; + if (PyLong_CheckExact(obj)) + return _PyLong_FormatWriter(writer, obj, 10, 0); + else + return format_obj(obj, writer); } /* parse the format_spec */ @@ -1382,7 +1430,7 @@ case 'n': /* no type conversion needed, already an int (or long). do the formatting */ - result = format_int_or_long_internal(obj, &format, tostring); + result = format_long_internal(obj, &format, writer); break; case 'e': @@ -1396,7 +1444,7 @@ tmp = PyNumber_Float(obj); if (tmp == NULL) goto done; - result = format_float_internal(tmp, &format); + result = format_float_internal(tmp, &format, writer); break; default: @@ -1407,41 +1455,27 @@ done: Py_XDECREF(tmp); + Py_XDECREF(str); return result; } -/* Need to define long_format as a function that will convert a long - to a string. In 3.0, _PyLong_Format has the correct signature. */ -#define long_format _PyLong_Format - -PyObject * -_PyLong_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +_PyFloat_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - return format_int_or_long(obj, format_spec, start, end, - long_format); -} - -PyObject * -_PyFloat_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) -{ - PyObject *result = NULL; InternalFormatSpec format; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) { - result = PyObject_Str(obj); - goto done; - } + if (start == end) + return format_obj(obj, writer); /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, '\0', '>')) - goto done; + return -1; /* type conversion? */ switch (format.type) { @@ -1455,38 +1489,32 @@ case 'n': case '%': /* no conversion, already a float. do the formatting */ - result = format_float_internal(obj, &format); - break; + return format_float_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; + return -1; } - -done: - return result; } -PyObject * -_PyComplex_FormatAdvanced(PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, Py_ssize_t end) +int +_PyComplex_FormatAdvancedWriter(_PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, Py_ssize_t end) { - PyObject *result = NULL; InternalFormatSpec format; /* check for the special case of zero length format spec, make it equivalent to str(obj) */ - if (start == end) { - result = PyObject_Str(obj); - goto done; - } + if (start == end) + return format_obj(obj, writer); /* parse the format_spec */ if (!parse_internal_render_format_spec(format_spec, start, end, &format, '\0', '>')) - goto done; + return -1; /* type conversion? */ switch (format.type) { @@ -1499,15 +1527,11 @@ case 'G': case 'n': /* no conversion, already a complex. do the formatting */ - result = format_complex_internal(obj, &format); - break; + return format_complex_internal(obj, &format, writer); default: /* unknown */ unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; + return -1; } - -done: - return result; } -- Repository URL: http://hg.python.org/cpython

1 0

cpython: PyArg_Parse*("U"): ensure that the Unicode string is ready
by victor.stinner May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/1700fb28d2ab changeset: 77222:1700fb28d2ab user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue May 29 12:30:29 2012 +0200 summary: PyArg_Parse*("U"): ensure that the Unicode string is ready files: Python/getargs.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Python/getargs.c b/Python/getargs.c --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1167,8 +1167,11 @@ case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - if (PyUnicode_Check(arg)) + if (PyUnicode_Check(arg)) { + if (PyUnicode_READY(arg) == -1) + RETURN_ERR_OCCURRED; *p = arg; + } else return converterr("str", arg, msgbuf, bufsize); break; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Use Python 3.x-style keyword only arg in Array()
by richard.oudkerk May 29, 2012

May 29, 2012

http://hg.python.org/cpython/rev/b31ea7995214 changeset: 77221:b31ea7995214 user: Richard Oudkerk <shibturn(a)gmail.com> date: Tue May 29 12:01:47 2012 +0100 summary: Use Python 3.x-style keyword only arg in Array() Previously a Python 2.x compatible hack was used for multiprocessing.sharedctypes.Array(). Also the documented signature was wrong. files: Doc/library/multiprocessing.rst | 6 +++--- Lib/multiprocessing/__init__.py | 8 ++++---- Lib/multiprocessing/sharedctypes.py | 7 ++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -953,7 +953,7 @@ It is possible to create shared objects using shared memory which can be inherited by child processes. -.. function:: Value(typecode_or_type, *args[, lock]) +.. function:: Value(typecode_or_type, *args, lock=True) Return a :mod:`ctypes` object allocated from shared memory. By default the return value is actually a synchronized wrapper for the object. @@ -1045,7 +1045,7 @@ attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *args[, lock]) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1060,7 +1060,7 @@ Note that *lock* is a keyword-only argument. -.. function:: Value(typecode_or_type, *args[, lock]) +.. function:: Value(typecode_or_type, *args, lock=True) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py --- a/Lib/multiprocessing/__init__.py +++ b/Lib/multiprocessing/__init__.py @@ -228,19 +228,19 @@ from multiprocessing.sharedctypes import RawArray return RawArray(typecode_or_type, size_or_initializer) -def Value(typecode_or_type, *args, **kwds): +def Value(typecode_or_type, *args, lock=True): ''' Returns a synchronized shared object ''' from multiprocessing.sharedctypes import Value - return Value(typecode_or_type, *args, **kwds) + return Value(typecode_or_type, *args, lock=lock) -def Array(typecode_or_type, size_or_initializer, **kwds): +def Array(typecode_or_type, size_or_initializer, *, lock=True): ''' Returns a synchronized shared array ''' from multiprocessing.sharedctypes import Array - return Array(typecode_or_type, size_or_initializer, **kwds) + return Array(typecode_or_type, size_or_initializer, lock=lock) # # diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py --- a/Lib/multiprocessing/sharedctypes.py +++ b/Lib/multiprocessing/sharedctypes.py @@ -63,7 +63,7 @@ result.__init__(*size_or_initializer) return result -def Value(typecode_or_type, *args, lock=None): +def Value(typecode_or_type, *args, lock=True): ''' Return a synchronization wrapper for a Value ''' @@ -76,13 +76,10 @@ raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock) -def Array(typecode_or_type, size_or_initializer, **kwds): +def Array(typecode_or_type, size_or_initializer, *, lock=True): ''' Return a synchronization wrapper for a RawArray ''' - lock = kwds.pop('lock', None) - if kwds: - raise ValueError('unrecognized keyword argument(s): %s' % list(kwds.keys())) obj = RawArray(typecode_or_type, size_or_initializer) if lock is False: return obj -- Repository URL: http://hg.python.org/cpython

1 0