Mailman 3 November 2013 - Python-checkins

cpython: Encapsulate cpickle global state in a dedicated object.
by alexandre.vassalotti 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/64c6d52793be changeset: 87620:64c6d52793be user: Alexandre Vassalotti <alexandre(a)peadrop.com> date: Wed Nov 27 19:36:52 2013 -0800 summary: Encapsulate cpickle global state in a dedicated object. This implements PEP 3121 module finalization as well. This change does not cause any significant impact on performance. files: Modules/_pickle.c | 716 ++++++++++++++++++++------------- 1 files changed, 436 insertions(+), 280 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1,6 +1,9 @@ #include "Python.h" #include "structmember.h" +PyDoc_STRVAR(pickle_module_doc, +"Optimized C implementation for the Python pickle module."); + /*[clinic] module _pickle class _pickle.Pickler @@ -25,9 +28,6 @@ [python]*/ /*[python checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ -PyDoc_STRVAR(pickle_module_doc, -"Optimized C implementation for the Python pickle module."); - /* Bump this when new opcodes are added to the pickle protocol. */ enum { HIGHEST_PROTOCOL = 4, @@ -132,40 +132,260 @@ FRAME_HEADER_SIZE = 9 }; -/* Exception classes for pickle. These should override the ones defined in - pickle.py, when the C-optimized Pickler and Unpickler are used. */ -static PyObject *PickleError = NULL; -static PyObject *PicklingError = NULL; -static PyObject *UnpicklingError = NULL; - -/* copyreg.dispatch_table, {type_object: pickling_function} */ -static PyObject *dispatch_table = NULL; -/* For EXT[124] opcodes. */ -/* copyreg._extension_registry, {(module_name, function_name): code} */ -static PyObject *extension_registry = NULL; -/* copyreg._inverted_registry, {code: (module_name, function_name)} */ -static PyObject *inverted_registry = NULL; -/* copyreg._extension_cache, {code: object} */ -static PyObject *extension_cache = NULL; - -/* _compat_pickle.NAME_MAPPING, {(oldmodule, oldname): (newmodule, newname)} */ -static PyObject *name_mapping_2to3 = NULL; -/* _compat_pickle.IMPORT_MAPPING, {oldmodule: newmodule} */ -static PyObject *import_mapping_2to3 = NULL; -/* Same, but with REVERSE_NAME_MAPPING / REVERSE_IMPORT_MAPPING */ -static PyObject *name_mapping_3to2 = NULL; -static PyObject *import_mapping_3to2 = NULL; - -/* XXX: Are these really nescessary? */ -/* As the name says, an empty tuple. */ -static PyObject *empty_tuple = NULL; -/* For looking up name pairs in copyreg._extension_registry. */ -static PyObject *two_tuple = NULL; +/*************************************************************************/ + +/* State of the pickle module, per PEP 3121. */ +typedef struct { + /* Exception classes for pickle. */ + PyObject *PickleError; + PyObject *PicklingError; + PyObject *UnpicklingError; + + /* copyreg.dispatch_table, {type_object: pickling_function} */ + PyObject *dispatch_table; + + /* For the extension opcodes EXT1, EXT2 and EXT4. */ + + /* copyreg._extension_registry, {(module_name, function_name): code} */ + PyObject *extension_registry; + /* copyreg._extension_cache, {code: object} */ + PyObject *extension_cache; + /* copyreg._inverted_registry, {code: (module_name, function_name)} */ + PyObject *inverted_registry; + + /* Import mappings for compatibility with Python 2.x */ + + /* _compat_pickle.NAME_MAPPING, + {(oldmodule, oldname): (newmodule, newname)} */ + PyObject *name_mapping_2to3; + /* _compat_pickle.IMPORT_MAPPING, {oldmodule: newmodule} */ + PyObject *import_mapping_2to3; + /* Same, but with REVERSE_NAME_MAPPING / REVERSE_IMPORT_MAPPING */ + PyObject *name_mapping_3to2; + PyObject *import_mapping_3to2; + + /* codecs.encode, used for saving bytes in older protocols */ + PyObject *codecs_encode; + + /* As the name says, an empty tuple. */ + PyObject *empty_tuple; + /* Single argument tuple used by _Pickle_FastCall */ + PyObject *arg_tuple; +} PickleState; + +/* Forward declaration of the _pickle module definition. */ +static struct PyModuleDef _picklemodule; + +/* Given a module object, get its per-module state. */ +static PickleState * +_Pickle_GetState(PyObject *module) +{ + return (PickleState *)PyModule_GetState(module); +} + +/* Find the module instance imported in the currently running sub-interpreter + and get its state. */ +static PickleState * +_Pickle_GetGlobalState(void) +{ + return _Pickle_GetState(PyState_FindModule(&_picklemodule)); +} + +/* Clear the given pickle module state. */ +static void +_Pickle_ClearState(PickleState *st) +{ + Py_CLEAR(st->PickleError); + Py_CLEAR(st->PicklingError); + Py_CLEAR(st->UnpicklingError); + Py_CLEAR(st->dispatch_table); + Py_CLEAR(st->extension_registry); + Py_CLEAR(st->extension_cache); + Py_CLEAR(st->inverted_registry); + Py_CLEAR(st->name_mapping_2to3); + Py_CLEAR(st->import_mapping_2to3); + Py_CLEAR(st->name_mapping_3to2); + Py_CLEAR(st->import_mapping_3to2); + Py_CLEAR(st->codecs_encode); + Py_CLEAR(st->empty_tuple); + Py_CLEAR(st->arg_tuple); +} + +/* Initialize the given pickle module state. */ +static int +_Pickle_InitState(PickleState *st) +{ + PyObject *copyreg = NULL; + PyObject *compat_pickle = NULL; + PyObject *codecs = NULL; + + copyreg = PyImport_ImportModule("copyreg"); + if (!copyreg) + goto error; + st->dispatch_table = PyObject_GetAttrString(copyreg, "dispatch_table"); + if (!st->dispatch_table) + goto error; + if (!PyDict_CheckExact(st->dispatch_table)) { + PyErr_Format(PyExc_RuntimeError, + "copyreg.dispatch_table should be a dict, not %.200s", + Py_TYPE(st->dispatch_table)->tp_name); + goto error; + } + st->extension_registry = \ + PyObject_GetAttrString(copyreg, "_extension_registry"); + if (!st->extension_registry) + goto error; + if (!PyDict_CheckExact(st->extension_registry)) { + PyErr_Format(PyExc_RuntimeError, + "copyreg._extension_registry should be a dict, " + "not %.200s", Py_TYPE(st->extension_registry)->tp_name); + goto error; + } + st->inverted_registry = \ + PyObject_GetAttrString(copyreg, "_inverted_registry"); + if (!st->inverted_registry) + goto error; + if (!PyDict_CheckExact(st->inverted_registry)) { + PyErr_Format(PyExc_RuntimeError, + "copyreg._inverted_registry should be a dict, " + "not %.200s", Py_TYPE(st->inverted_registry)->tp_name); + goto error; + } + st->extension_cache = PyObject_GetAttrString(copyreg, "_extension_cache"); + if (!st->extension_cache) + goto error; + if (!PyDict_CheckExact(st->extension_cache)) { + PyErr_Format(PyExc_RuntimeError, + "copyreg._extension_cache should be a dict, " + "not %.200s", Py_TYPE(st->extension_cache)->tp_name); + goto error; + } + Py_CLEAR(copyreg); + + /* Load the 2.x -> 3.x stdlib module mapping tables */ + compat_pickle = PyImport_ImportModule("_compat_pickle"); + if (!compat_pickle) + goto error; + st->name_mapping_2to3 = \ + PyObject_GetAttrString(compat_pickle, "NAME_MAPPING"); + if (!st->name_mapping_2to3) + goto error; + if (!PyDict_CheckExact(st->name_mapping_2to3)) { + PyErr_Format(PyExc_RuntimeError, + "_compat_pickle.NAME_MAPPING should be a dict, not %.200s", + Py_TYPE(st->name_mapping_2to3)->tp_name); + goto error; + } + st->import_mapping_2to3 = \ + PyObject_GetAttrString(compat_pickle, "IMPORT_MAPPING"); + if (!st->import_mapping_2to3) + goto error; + if (!PyDict_CheckExact(st->import_mapping_2to3)) { + PyErr_Format(PyExc_RuntimeError, + "_compat_pickle.IMPORT_MAPPING should be a dict, " + "not %.200s", Py_TYPE(st->import_mapping_2to3)->tp_name); + goto error; + } + /* ... and the 3.x -> 2.x mapping tables */ + st->name_mapping_3to2 = \ + PyObject_GetAttrString(compat_pickle, "REVERSE_NAME_MAPPING"); + if (!st->name_mapping_3to2) + goto error; + if (!PyDict_CheckExact(st->name_mapping_3to2)) { + PyErr_Format(PyExc_RuntimeError, + "_compat_pickle.REVERSE_NAME_MAPPING should be a dict, " + "not %.200s", Py_TYPE(st->name_mapping_3to2)->tp_name); + goto error; + } + st->import_mapping_3to2 = \ + PyObject_GetAttrString(compat_pickle, "REVERSE_IMPORT_MAPPING"); + if (!st->import_mapping_3to2) + goto error; + if (!PyDict_CheckExact(st->import_mapping_3to2)) { + PyErr_Format(PyExc_RuntimeError, + "_compat_pickle.REVERSE_IMPORT_MAPPING should be a dict, " + "not %.200s", Py_TYPE(st->import_mapping_3to2)->tp_name); + goto error; + } + Py_CLEAR(compat_pickle); + + codecs = PyImport_ImportModule("codecs"); + if (codecs == NULL) + goto error; + st->codecs_encode = PyObject_GetAttrString(codecs, "encode"); + if (st->codecs_encode == NULL) { + goto error; + } + if (!PyCallable_Check(st->codecs_encode)) { + PyErr_Format(PyExc_RuntimeError, + "codecs.encode should be a callable, not %.200s", + Py_TYPE(st->codecs_encode)->tp_name); + goto error; + } + Py_CLEAR(codecs); + + st->empty_tuple = PyTuple_New(0); + if (st->empty_tuple == NULL) + goto error; + + st->arg_tuple = NULL; + + return 0; + + error: + Py_CLEAR(copyreg); + Py_CLEAR(compat_pickle); + Py_CLEAR(codecs); + _Pickle_ClearState(st); + return -1; +} + +/* Helper for calling a function with a single argument quickly. + + This has the performance advantage of reusing the argument tuple. This + provides a nice performance boost with older pickle protocols where many + unbuffered reads occurs. + + This function steals the reference of the given argument. */ +static PyObject * +_Pickle_FastCall(PyObject *func, PyObject *obj) +{ + PickleState *st = _Pickle_GetGlobalState(); + PyObject *arg_tuple; + PyObject *result; + + arg_tuple = st->arg_tuple; + if (arg_tuple == NULL) { + arg_tuple = PyTuple_New(1); + if (arg_tuple == NULL) { + Py_DECREF(obj); + return NULL; + } + } + assert(arg_tuple->ob_refcnt == 1); + assert(PyTuple_GET_ITEM(arg_tuple, 0) == NULL); + + PyTuple_SET_ITEM(arg_tuple, 0, obj); + result = PyObject_Call(func, arg_tuple, NULL); + + Py_CLEAR(PyTuple_GET_ITEM(arg_tuple, 0)); + if (arg_tuple->ob_refcnt > 1) { + /* The function called saved a reference to our argument tuple. + This means we cannot reuse it anymore. */ + Py_CLEAR(arg_tuple); + } + st->arg_tuple = arg_tuple; + + return result; +} + +/*************************************************************************/ static int stack_underflow(void) { - PyErr_SetString(UnpicklingError, "unpickling stack underflow"); + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "unpickling stack underflow"); return -1; } @@ -266,8 +486,9 @@ static PyObject * Pdata_pop(Pdata *self) { + PickleState *st = _Pickle_GetGlobalState(); if (Py_SIZE(self) == 0) { - PyErr_SetString(UnpicklingError, "bad pickle data"); + PyErr_SetString(st->UnpicklingError, "bad pickle data"); return NULL; } return self->data[--Py_SIZE(self)]; @@ -637,40 +858,6 @@ /*************************************************************************/ -/* Helper for calling a function with a single argument quickly. - - This has the performance advantage of reusing the argument tuple. This - provides a nice performance boost with older pickle protocols where many - unbuffered reads occurs. - - This function steals the reference of the given argument. */ -static PyObject * -_Pickle_FastCall(PyObject *func, PyObject *obj) -{ - static PyObject *arg_tuple = NULL; - PyObject *result; - - if (arg_tuple == NULL) { - arg_tuple = PyTuple_New(1); - if (arg_tuple == NULL) { - Py_DECREF(obj); - return NULL; - } - } - assert(arg_tuple->ob_refcnt == 1); - assert(PyTuple_GET_ITEM(arg_tuple, 0) == NULL); - - PyTuple_SET_ITEM(arg_tuple, 0, obj); - result = PyObject_Call(func, arg_tuple, NULL); - - Py_CLEAR(PyTuple_GET_ITEM(arg_tuple, 0)); - if (arg_tuple->ob_refcnt > 1) { - /* The function called saved a reference to our argument tuple. - This means we cannot reuse it anymore. */ - Py_CLEAR(arg_tuple); - } - return result; -} static int _Pickler_ClearBuffer(PicklerObject *self) @@ -963,8 +1150,10 @@ if (_Unpickler_SkipConsumed(self) < 0) return -1; - if (n == READ_WHOLE_LINE) - data = PyObject_Call(self->readline, empty_tuple, NULL); + if (n == READ_WHOLE_LINE) { + PickleState *st = _Pickle_GetGlobalState(); + data = PyObject_Call(self->readline, st->empty_tuple, NULL); + } else { PyObject *len = PyLong_FromSsize_t(n); if (len == NULL) @@ -1308,7 +1497,8 @@ len = 5; } else { /* unlikely */ - PyErr_SetString(PicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINGET"); return -1; } @@ -1364,7 +1554,8 @@ len = 5; } else { /* unlikely */ - PyErr_SetString(PicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINPUT"); return -1; } @@ -1807,26 +1998,14 @@ Python 2 *and* the appropriate 'bytes' object when unpickled using Python 3. Again this is a hack and we don't need to do this with newer protocols. */ - static PyObject *codecs_encode = NULL; PyObject *reduce_value = NULL; int status; - if (codecs_encode == NULL) { - PyObject *codecs_module = PyImport_ImportModule("codecs"); - if (codecs_module == NULL) { - return -1; - } - codecs_encode = PyObject_GetAttrString(codecs_module, "encode"); - Py_DECREF(codecs_module); - if (codecs_encode == NULL) { - return -1; - } - } - if (PyBytes_GET_SIZE(obj) == 0) { reduce_value = Py_BuildValue("(O())", (PyObject*)&PyBytes_Type); } else { + PickleState *st = _Pickle_GetGlobalState(); PyObject *unicode_str = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj), @@ -1836,7 +2015,7 @@ if (unicode_str == NULL) return -1; reduce_value = Py_BuildValue("(O(OO))", - codecs_encode, unicode_str, + st->codecs_encode, unicode_str, _PyUnicode_FromId(&PyId_latin1)); Py_DECREF(unicode_str); } @@ -2840,11 +3019,12 @@ { PyObject *key; PyObject *item; + PickleState *st = _Pickle_GetGlobalState(); key = PyTuple_Pack(2, *module_name, *global_name); if (key == NULL) return -1; - item = PyDict_GetItemWithError(name_mapping_3to2, key); + item = PyDict_GetItemWithError(st->name_mapping_3to2, key); Py_DECREF(key); if (item) { PyObject *fixed_module_name; @@ -2880,7 +3060,7 @@ return -1; } - item = PyDict_GetItemWithError(import_mapping_3to2, *module_name); + item = PyDict_GetItemWithError(st->import_mapping_3to2, *module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -2907,6 +3087,7 @@ PyObject *module_name = NULL; PyObject *module = NULL; PyObject *cls; + PickleState *st = _Pickle_GetGlobalState(); int status = 0; _Py_IDENTIFIER(__name__); _Py_IDENTIFIER(__qualname__); @@ -2947,21 +3128,21 @@ extra parameters of __import__ to fix that. */ module = PyImport_Import(module_name); if (module == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: import of module %R failed", obj, module_name); goto error; } cls = getattribute(module, global_name, self->proto >= 4); if (cls == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", obj, global_name, module_name); goto error; } if (cls != obj) { Py_DECREF(cls); - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: it's not the same object as %S.%S", obj, module_name, global_name); goto error; @@ -2972,14 +3153,18 @@ /* See whether this is in the extension registry, and if * so generate an EXT opcode. */ + PyObject *extension_key; PyObject *code_obj; /* extension code as Python object */ long code; /* extension code as C value */ char pdata[5]; Py_ssize_t n; - PyTuple_SET_ITEM(two_tuple, 0, module_name); - PyTuple_SET_ITEM(two_tuple, 1, global_name); - code_obj = PyDict_GetItem(extension_registry, two_tuple); + extension_key = PyTuple_Pack(2, module_name, global_name); + if (extension_key == NULL) { + goto error; + } + code_obj = PyDict_GetItem(st->extension_registry, extension_key); + Py_DECREF(extension_key); /* The object is not registered in the extension registry. This is the most likely code path. */ if (code_obj == NULL) @@ -2991,7 +3176,7 @@ /* Verify code_obj has the right type and value. */ if (!PyLong_Check(code_obj)) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: extension code %R isn't an integer", obj, code_obj); goto error; @@ -2999,9 +3184,8 @@ code = PyLong_AS_LONG(code_obj); if (code <= 0 || code > 0x7fffffffL) { if (!PyErr_Occurred()) - PyErr_Format(PicklingError, - "Can't pickle %R: extension code %ld is out of range", - obj, code); + PyErr_Format(st->PicklingError, "Can't pickle %R: extension " + "code %ld is out of range", obj, code); goto error; } @@ -3074,7 +3258,7 @@ encoded = unicode_encoder(module_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "can't pickle module identifier '%S' using " "pickle protocol %i", module_name, self->proto); @@ -3093,7 +3277,7 @@ encoded = unicode_encoder(global_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "can't pickle global identifier '%S' using " "pickle protocol %i", global_name, self->proto); @@ -3206,6 +3390,7 @@ PyObject *state = NULL; PyObject *listitems = Py_None; PyObject *dictitems = Py_None; + PickleState *st = _Pickle_GetGlobalState(); Py_ssize_t size; int use_newobj = 0, use_newobj_ex = 0; @@ -3216,7 +3401,7 @@ size = PyTuple_Size(args); if (size < 2 || size > 5) { - PyErr_SetString(PicklingError, "tuple returned by " + PyErr_SetString(st->PicklingError, "tuple returned by " "__reduce__ must contain 2 through 5 elements"); return -1; } @@ -3226,12 +3411,12 @@ return -1; if (!PyCallable_Check(callable)) { - PyErr_SetString(PicklingError, "first item of the tuple " + PyErr_SetString(st->PicklingError, "first item of the tuple " "returned by __reduce__ must be callable"); return -1; } if (!PyTuple_Check(argtup)) { - PyErr_SetString(PicklingError, "second item of the tuple " + PyErr_SetString(st->PicklingError, "second item of the tuple " "returned by __reduce__ must be a tuple"); return -1; } @@ -3242,7 +3427,7 @@ if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(PicklingError, "fourth element of the tuple " + PyErr_Format(st->PicklingError, "fourth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(listitems)->tp_name); return -1; @@ -3251,7 +3436,7 @@ if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(PicklingError, "fifth element of the tuple " + PyErr_Format(st->PicklingError, "fifth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(dictitems)->tp_name); return -1; @@ -3290,7 +3475,7 @@ PyObject *kwargs; if (Py_SIZE(argtup) != 3) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "length of the NEWOBJ_EX argument tuple must be " "exactly 3, not %zd", Py_SIZE(argtup)); return -1; @@ -3298,21 +3483,21 @@ cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "first item from NEWOBJ_EX argument tuple must " "be a class, not %.200s", Py_TYPE(cls)->tp_name); return -1; } args = PyTuple_GET_ITEM(argtup, 1); if (!PyTuple_Check(args)) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "second item from NEWOBJ_EX argument tuple must " "be a tuple, not %.200s", Py_TYPE(args)->tp_name); return -1; } kwargs = PyTuple_GET_ITEM(argtup, 2); if (!PyDict_Check(kwargs)) { - PyErr_Format(PicklingError, + PyErr_Format(st->PicklingError, "third item from NEWOBJ_EX argument tuple must " "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name); return -1; @@ -3333,13 +3518,13 @@ /* Sanity checks. */ if (Py_SIZE(argtup) < 1) { - PyErr_SetString(PicklingError, "__newobj__ arglist is empty"); + PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty"); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_SetString(PicklingError, "args[0] from " + PyErr_SetString(st->PicklingError, "args[0] from " "__newobj__ args is not a type"); return -1; } @@ -3349,7 +3534,7 @@ p = obj_class != cls; /* true iff a problem */ Py_DECREF(obj_class); if (p) { - PyErr_SetString(PicklingError, "args[0] from " + PyErr_SetString(st->PicklingError, "args[0] from " "__newobj__ args has the wrong class"); return -1; } @@ -3547,7 +3732,8 @@ * __reduce_ex__ method, or the object's __reduce__ method. */ if (self->dispatch_table == NULL) { - reduce_func = PyDict_GetItem(dispatch_table, (PyObject *)type); + PickleState *st = _Pickle_GetGlobalState(); + reduce_func = PyDict_GetItem(st->dispatch_table, (PyObject *)type); /* PyDict_GetItem() unlike PyObject_GetItem() and PyObject_GetAttr() returns a borrowed ref */ Py_XINCREF(reduce_func); @@ -3591,17 +3777,23 @@ } } else { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) + PickleState *st = _Pickle_GetGlobalState(); + + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); - else + } + else { goto error; + } /* Check for a __reduce__ method. */ reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__); if (reduce_func != NULL) { - reduce_value = PyObject_Call(reduce_func, empty_tuple, NULL); + reduce_value = PyObject_Call(reduce_func, st->empty_tuple, + NULL); } else { - PyErr_Format(PicklingError, "can't pickle '%.200s' object: %R", + PyErr_Format(st->PicklingError, + "can't pickle '%.200s' object: %R", type->tp_name, obj); goto error; } @@ -3617,7 +3809,8 @@ } if (!PyTuple_Check(reduce_value)) { - PyErr_SetString(PicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->PicklingError, "__reduce__ must return a string or tuple"); goto error; } @@ -3723,7 +3916,8 @@ Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(PicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_Format(st->PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -3919,16 +4113,19 @@ if (self->dispatch_table == NULL) return NULL; } - return Py_None; -} - -/* XXX Slight hack to slot a Clinic generated signature in tp_init. */ + + Py_RETURN_NONE; +} + +/* Wrap the Clinic generated signature to slot it in tp_init. */ static int Pickler_init(PyObject *self, PyObject *args, PyObject *kwargs) { - if (_pickle_Pickler___init__(self, args, kwargs) == NULL) { - return -1; - } + PyObject *result = _pickle_Pickler___init__(self, args, kwargs); + if (result == NULL) { + return -1; + } + Py_DECREF(result); return 0; } @@ -4323,8 +4520,9 @@ static Py_ssize_t marker(UnpicklerObject *self) { + PickleState *st = _Pickle_GetGlobalState(); if (self->num_marks < 1) { - PyErr_SetString(UnpicklingError, "could not find MARK"); + PyErr_SetString(st->UnpicklingError, "could not find MARK"); return -1; } @@ -4341,7 +4539,8 @@ static int bad_readline(void) { - PyErr_SetString(UnpicklingError, "pickle data was truncated"); + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "pickle data was truncated"); return -1; } @@ -4536,8 +4735,9 @@ size = calc_binint(nbytes, size); if (size < 0) { + PickleState *st = _Pickle_GetGlobalState(); /* Corrupt or hostile pickle -- we never write one like this */ - PyErr_SetString(UnpicklingError, + PyErr_SetString(st->UnpicklingError, "LONG pickle has negative byte count"); return -1; } @@ -4625,7 +4825,8 @@ len -= 2; } else { - PyErr_SetString(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "the STRING opcode argument must be quoted"); return -1; } @@ -4686,7 +4887,8 @@ size = calc_binsize(s, nbytes); if (size < 0) { - PyErr_Format(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_Format(st->UnpicklingError, "BINSTRING exceeds system's maximum size of %zd bytes", PY_SSIZE_T_MAX); return -1; @@ -4994,6 +5196,7 @@ PyObject *clsraw = NULL; PyTypeObject *cls; /* clsraw cast to its true type */ PyObject *obj; + PickleState *st = _Pickle_GetGlobalState(); /* Stack is ... cls argtuple, and we want to call * cls.__new__(cls, *argtuple). @@ -5002,7 +5205,8 @@ if (args == NULL) goto error; if (!PyTuple_Check(args)) { - PyErr_SetString(UnpicklingError, "NEWOBJ expected an arg " "tuple."); + PyErr_SetString(st->UnpicklingError, + "NEWOBJ expected an arg " "tuple."); goto error; } @@ -5011,12 +5215,12 @@ if (cls == NULL) goto error; if (!PyType_Check(cls)) { - PyErr_SetString(UnpicklingError, "NEWOBJ class argument " + PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument " "isn't a type object"); goto error; } if (cls->tp_new == NULL) { - PyErr_SetString(UnpicklingError, "NEWOBJ class argument " + PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument " "has NULL tp_new"); goto error; } @@ -5042,6 +5246,7 @@ { PyObject *cls, *args, *kwargs; PyObject *obj; + PickleState *st = _Pickle_GetGlobalState(); PDATA_POP(self->stack, kwargs); if (kwargs == NULL) { @@ -5063,7 +5268,7 @@ Py_DECREF(kwargs); Py_DECREF(args); Py_DECREF(cls); - PyErr_Format(UnpicklingError, + PyErr_Format(st->UnpicklingError, "NEWOBJ_EX class argument must be a type, not %.200s", Py_TYPE(cls)->tp_name); return -1; @@ -5073,7 +5278,7 @@ Py_DECREF(kwargs); Py_DECREF(args); Py_DECREF(cls); - PyErr_SetString(UnpicklingError, + PyErr_SetString(st->UnpicklingError, "NEWOBJ_EX class argument doesn't have __new__"); return -1; } @@ -5135,7 +5340,8 @@ PDATA_POP(self->stack, module_name); if (module_name == NULL || !PyUnicode_CheckExact(module_name) || global_name == NULL || !PyUnicode_CheckExact(global_name)) { - PyErr_SetString(UnpicklingError, "STACK_GLOBAL requires str"); + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str"); Py_XDECREF(global_name); Py_XDECREF(module_name); return -1; @@ -5176,7 +5382,8 @@ return 0; } else { - PyErr_SetString(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered,\n" "but no persistent_load function was specified."); return -1; @@ -5203,7 +5410,8 @@ return 0; } else { - PyErr_SetString(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered,\n" "but no persistent_load function was specified."); return -1; @@ -5359,6 +5567,7 @@ PyObject *obj; /* the object to push */ PyObject *pair; /* (module_name, class_name) */ PyObject *module_name, *class_name; + PickleState *st = _Pickle_GetGlobalState(); assert(nbytes == 1 || nbytes == 2 || nbytes == 4); if (_Unpickler_Read(self, &codebytes, nbytes) < 0) @@ -5366,7 +5575,7 @@ code = calc_binint(codebytes, nbytes); if (code <= 0) { /* note that 0 is forbidden */ /* Corrupt or hostile pickle. */ - PyErr_SetString(UnpicklingError, "EXT specifies code <= 0"); + PyErr_SetString(st->UnpicklingError, "EXT specifies code <= 0"); return -1; } @@ -5374,7 +5583,7 @@ py_code = PyLong_FromLong(code); if (py_code == NULL) return -1; - obj = PyDict_GetItem(extension_cache, py_code); + obj = PyDict_GetItem(st->extension_cache, py_code); if (obj != NULL) { /* Bingo. */ Py_DECREF(py_code); @@ -5383,7 +5592,7 @@ } /* Look up the (module_name, class_name) pair. */ - pair = PyDict_GetItem(inverted_registry, py_code); + pair = PyDict_GetItem(st->inverted_registry, py_code); if (pair == NULL) { Py_DECREF(py_code); PyErr_Format(PyExc_ValueError, "unregistered extension " @@ -5408,7 +5617,7 @@ return -1; } /* Cache code -> obj. */ - code = PyDict_SetItem(extension_cache, py_code, obj); + code = PyDict_SetItem(st->extension_cache, py_code, obj); Py_DECREF(py_code); if (code < 0) { Py_DECREF(obj); @@ -5585,8 +5794,10 @@ if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { + PickleState *st = _Pickle_GetGlobalState(); /* Currupt or hostile pickle -- we never write one like this. */ - PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS"); + PyErr_SetString(st->UnpicklingError, + "odd number of items for SETITEMS"); return -1; } @@ -5736,7 +5947,8 @@ _Py_IDENTIFIER(__dict__); if (!PyDict_Check(state)) { - PyErr_SetString(UnpicklingError, "state is not a dictionary"); + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "state is not a dictionary"); goto error; } dict = _PyObject_GetAttrId(inst, &PyId___dict__); @@ -5765,7 +5977,8 @@ Py_ssize_t i; if (!PyDict_Check(slotstate)) { - PyErr_SetString(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "slot state is not a dictionary"); goto error; } @@ -5988,11 +6201,14 @@ break; default: - if (s[0] == '\0') + if (s[0] == '\0') { PyErr_SetNone(PyExc_EOFError); - else - PyErr_Format(UnpicklingError, + } + else { + PickleState *st = _Pickle_GetGlobalState(); + PyErr_Format(st->UnpicklingError, "invalid load key, '%c'.", s[0]); + } return NULL; } @@ -6037,12 +6253,14 @@ /*[clinic checksum: 9a30ba4e4d9221d4dcd705e1471ab11b2c9e3ac6]*/ { UnpicklerObject *unpickler = (UnpicklerObject*)self; + /* Check whether the Unpickler was initialized correctly. This prevents segfaulting if a subclass overridden __init__ with a function that does not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (unpickler->read == NULL) { - PyErr_Format(UnpicklingError, + PickleState *st = _Pickle_GetGlobalState(); + PyErr_Format(st->UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(unpickler)->tp_name); return NULL; @@ -6121,13 +6339,14 @@ if (self->proto < 3 && self->fix_imports) { PyObject *key; PyObject *item; + PickleState *st = _Pickle_GetGlobalState(); /* Check if the global (i.e., a function or a class) was renamed or moved to another module. */ key = PyTuple_Pack(2, module_name, global_name); if (key == NULL) return NULL; - item = PyDict_GetItemWithError(name_mapping_2to3, key); + item = PyDict_GetItemWithError(st->name_mapping_2to3, key); Py_DECREF(key); if (item) { if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { @@ -6153,7 +6372,7 @@ } /* Check if the module was renamed. */ - item = PyDict_GetItemWithError(import_mapping_2to3, module_name); + item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -6378,16 +6597,18 @@ self->proto = 0; - return Py_None; -} - -/* XXX Slight hack to slot a Clinic generated signature in tp_init. */ + Py_RETURN_NONE; +} + +/* Wrap the Clinic generated signature to slot it in tp_init. */ static int Unpickler_init(PyObject *self, PyObject *args, PyObject *kwargs) { - if (_pickle_Unpickler___init__(self, args, kwargs) == NULL) { - return -1; - } + PyObject *result = _pickle_Unpickler___init__(self, args, kwargs); + if (result == NULL) { + return -1; + } + Py_DECREF(result); return 0; } @@ -7174,7 +7395,6 @@ return NULL; } - static struct PyMethodDef pickle_methods[] = { _PICKLE_DUMP_METHODDEF _PICKLE_DUMPS_METHODDEF @@ -7184,125 +7404,56 @@ }; static int -initmodule(void) -{ - PyObject *copyreg = NULL; - PyObject *compat_pickle = NULL; - - /* XXX: We should ensure that the types of the dictionaries imported are - exactly PyDict objects. Otherwise, it is possible to crash the pickle - since we use the PyDict API directly to access these dictionaries. */ - - copyreg = PyImport_ImportModule("copyreg"); - if (!copyreg) - goto error; - dispatch_table = PyObject_GetAttrString(copyreg, "dispatch_table"); - if (!dispatch_table) - goto error; - extension_registry = \ - PyObject_GetAttrString(copyreg, "_extension_registry"); - if (!extension_registry) - goto error; - inverted_registry = PyObject_GetAttrString(copyreg, "_inverted_registry"); - if (!inverted_registry) - goto error; - extension_cache = PyObject_GetAttrString(copyreg, "_extension_cache"); - if (!extension_cache) - goto error; - Py_CLEAR(copyreg); - - /* Load the 2.x -> 3.x stdlib module mapping tables */ - compat_pickle = PyImport_ImportModule("_compat_pickle"); - if (!compat_pickle) - goto error; - name_mapping_2to3 = PyObject_GetAttrString(compat_pickle, "NAME_MAPPING"); - if (!name_mapping_2to3) - goto error; - if (!PyDict_CheckExact(name_mapping_2to3)) { - PyErr_Format(PyExc_RuntimeError, - "_compat_pickle.NAME_MAPPING should be a dict, not %.200s", - Py_TYPE(name_mapping_2to3)->tp_name); - goto error; - } - import_mapping_2to3 = PyObject_GetAttrString(compat_pickle, - "IMPORT_MAPPING"); - if (!import_mapping_2to3) - goto error; - if (!PyDict_CheckExact(import_mapping_2to3)) { - PyErr_Format(PyExc_RuntimeError, - "_compat_pickle.IMPORT_MAPPING should be a dict, " - "not %.200s", Py_TYPE(import_mapping_2to3)->tp_name); - goto error; - } - /* ... and the 3.x -> 2.x mapping tables */ - name_mapping_3to2 = PyObject_GetAttrString(compat_pickle, - "REVERSE_NAME_MAPPING"); - if (!name_mapping_3to2) - goto error; - if (!PyDict_CheckExact(name_mapping_3to2)) { - PyErr_Format(PyExc_RuntimeError, - "_compat_pickle.REVERSE_NAME_MAPPING should be a dict, " - "not %.200s", Py_TYPE(name_mapping_3to2)->tp_name); - goto error; - } - import_mapping_3to2 = PyObject_GetAttrString(compat_pickle, - "REVERSE_IMPORT_MAPPING"); - if (!import_mapping_3to2) - goto error; - if (!PyDict_CheckExact(import_mapping_3to2)) { - PyErr_Format(PyExc_RuntimeError, - "_compat_pickle.REVERSE_IMPORT_MAPPING should be a dict, " - "not %.200s", Py_TYPE(import_mapping_3to2)->tp_name); - goto error; - } - Py_CLEAR(compat_pickle); - - empty_tuple = PyTuple_New(0); - if (empty_tuple == NULL) - goto error; - two_tuple = PyTuple_New(2); - if (two_tuple == NULL) - goto error; - /* We use this temp container with no regard to refcounts, or to - * keeping containees alive. Exempt from GC, because we don't - * want anything looking at two_tuple() by magic. - */ - PyObject_GC_UnTrack(two_tuple); - +pickle_clear(PyObject *m) +{ + _Pickle_ClearState(_Pickle_GetState(m)); return 0; - - error: - Py_CLEAR(copyreg); - Py_CLEAR(dispatch_table); - Py_CLEAR(extension_registry); - Py_CLEAR(inverted_registry); - Py_CLEAR(extension_cache); - Py_CLEAR(compat_pickle); - Py_CLEAR(name_mapping_2to3); - Py_CLEAR(import_mapping_2to3); - Py_CLEAR(name_mapping_3to2); - Py_CLEAR(import_mapping_3to2); - Py_CLEAR(empty_tuple); - Py_CLEAR(two_tuple); - return -1; +} + +static int +pickle_traverse(PyObject *m, visitproc visit, void *arg) +{ + PickleState *st = _Pickle_GetState(m); + Py_VISIT(st->PickleError); + Py_VISIT(st->PicklingError); + Py_VISIT(st->UnpicklingError); + Py_VISIT(st->dispatch_table); + Py_VISIT(st->extension_registry); + Py_VISIT(st->extension_cache); + Py_VISIT(st->inverted_registry); + Py_VISIT(st->name_mapping_2to3); + Py_VISIT(st->import_mapping_2to3); + Py_VISIT(st->name_mapping_3to2); + Py_VISIT(st->import_mapping_3to2); + Py_VISIT(st->codecs_encode); + Py_VISIT(st->empty_tuple); + Py_VISIT(st->arg_tuple); + return 0; } static struct PyModuleDef _picklemodule = { PyModuleDef_HEAD_INIT, - "_pickle", - pickle_module_doc, - -1, - pickle_methods, - NULL, - NULL, - NULL, - NULL + "_pickle", /* m_name */ + pickle_module_doc, /* m_doc */ + sizeof(PickleState), /* m_size */ + pickle_methods, /* m_methods */ + NULL, /* m_reload */ + pickle_traverse, /* m_traverse */ + pickle_clear, /* m_clear */ + NULL /* m_free */ }; PyMODINIT_FUNC PyInit__pickle(void) { PyObject *m; + PickleState *st; + + m = PyState_FindModule(&_picklemodule); + if (m) { + Py_INCREF(m); + return m; + } if (PyType_Ready(&Unpickler_Type) < 0) return NULL; @@ -7327,27 +7478,32 @@ if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0) return NULL; + st = _Pickle_GetState(m); + /* Initialize the exceptions. */ - PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL); - if (PickleError == NULL) + st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL); + if (st->PickleError == NULL) return NULL; - PicklingError = \ - PyErr_NewException("_pickle.PicklingError", PickleError, NULL); - if (PicklingError == NULL) + st->PicklingError = \ + PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL); + if (st->PicklingError == NULL) return NULL; - UnpicklingError = \ - PyErr_NewException("_pickle.UnpicklingError", PickleError, NULL); - if (UnpicklingError == NULL) + st->UnpicklingError = \ + PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL); + if (st->UnpicklingError == NULL) return NULL; - if (PyModule_AddObject(m, "PickleError", PickleError) < 0) + Py_INCREF(st->PickleError); + if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) return NULL; - if (PyModule_AddObject(m, "PicklingError", PicklingError) < 0) + Py_INCREF(st->PicklingError); + if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) return NULL; - if (PyModule_AddObject(m, "UnpicklingError", UnpicklingError) < 0) + Py_INCREF(st->UnpicklingError); + if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) return NULL; - if (initmodule() < 0) + if (_Pickle_InitState(st) < 0) return NULL; return m; -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: Change write buffer use to avoid O(N**2). Make write()/sendto() accept
by guido.van.rossum 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/80e0040d910c changeset: 87617:80e0040d910c user: Guido van Rossum <guido(a)python.org> date: Wed Nov 27 14:12:48 2013 -0800 summary: asyncio: Change write buffer use to avoid O(N**2). Make write()/sendto() accept bytearray/memoryview too. Change some asserts with proper exceptions. files: Lib/asyncio/selector_events.py | 82 ++- Lib/test/test_asyncio/test_selector_events.py | 203 +++++++-- 2 files changed, 207 insertions(+), 78 deletions(-) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -340,6 +340,8 @@ max_size = 256 * 1024 # Buffer size passed to recv(). + _buffer_factory = bytearray # Constructs initial value for self._buffer. + def __init__(self, loop, sock, protocol, extra, server=None): super().__init__(extra) self._extra['socket'] = sock @@ -354,7 +356,7 @@ self._sock_fd = sock.fileno() self._protocol = protocol self._server = server - self._buffer = collections.deque() + self._buffer = self._buffer_factory() self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. self._protocol_paused = False @@ -433,12 +435,14 @@ high = 4*low if low is None: low = high // 4 - assert 0 <= low <= high, repr((low, high)) + if not high >= low >= 0: + raise ValueError('high (%r) must be >= low (%r) must be >= 0' % + (high, low)) self._high_water = high self._low_water = low def get_write_buffer_size(self): - return sum(len(data) for data in self._buffer) + return len(self._buffer) class _SelectorSocketTransport(_SelectorTransport): @@ -455,13 +459,16 @@ self._loop.call_soon(waiter.set_result, None) def pause_reading(self): - assert not self._closing, 'Cannot pause_reading() when closing' - assert not self._paused, 'Already paused' + if self._closing: + raise RuntimeError('Cannot pause_reading() when closing') + if self._paused: + raise RuntimeError('Already paused') self._paused = True self._loop.remove_reader(self._sock_fd) def resume_reading(self): - assert self._paused, 'Not paused' + if not self._paused: + raise RuntimeError('Not paused') self._paused = False if self._closing: return @@ -488,8 +495,11 @@ self.close() def write(self, data): - assert isinstance(data, bytes), repr(type(data)) - assert not self._eof, 'Cannot call write() after write_eof()' + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) + if self._eof: + raise RuntimeError('Cannot call write() after write_eof()') if not data: return @@ -516,25 +526,23 @@ self._loop.add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. - self._buffer.append(data) + self._buffer.extend(data) self._maybe_pause_protocol() def _write_ready(self): - data = b''.join(self._buffer) - assert data, 'Data should not be empty' + assert self._buffer, 'Data should not be empty' - self._buffer.clear() # Optimistically; may have to put it back later. try: - n = self._sock.send(data) + n = self._sock.send(self._buffer) except (BlockingIOError, InterruptedError): - self._buffer.append(data) # Still need to write this. + pass except Exception as exc: self._loop.remove_writer(self._sock_fd) + self._buffer.clear() self._fatal_error(exc) else: - data = data[n:] - if data: - self._buffer.append(data) # Still need to write this. + if n: + del self._buffer[:n] self._maybe_resume_protocol() # May append to buffer. if not self._buffer: self._loop.remove_writer(self._sock_fd) @@ -556,6 +564,8 @@ class _SelectorSslTransport(_SelectorTransport): + _buffer_factory = bytearray + def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None, server_side=False, server_hostname=None, extra=None, server=None): @@ -661,13 +671,16 @@ # accept more data for the buffer and eventually the app will # call resume_reading() again, and things will flow again. - assert not self._closing, 'Cannot pause_reading() when closing' - assert not self._paused, 'Already paused' + if self._closing: + raise RuntimeError('Cannot pause_reading() when closing') + if self._paused: + raise RuntimeError('Already paused') self._paused = True self._loop.remove_reader(self._sock_fd) def resume_reading(self): - assert self._paused, 'Not paused' + if not self._paused: + raise ('Not paused') self._paused = False if self._closing: return @@ -712,10 +725,8 @@ self._loop.add_reader(self._sock_fd, self._read_ready) if self._buffer: - data = b''.join(self._buffer) - self._buffer.clear() try: - n = self._sock.send(data) + n = self._sock.send(self._buffer) except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError): n = 0 @@ -725,11 +736,12 @@ self._write_wants_read = True except Exception as exc: self._loop.remove_writer(self._sock_fd) + self._buffer.clear() self._fatal_error(exc) return - if n < len(data): - self._buffer.append(data[n:]) + if n: + del self._buffer[:n] self._maybe_resume_protocol() # May append to buffer. @@ -739,7 +751,9 @@ self._call_connection_lost(None) def write(self, data): - assert isinstance(data, bytes), repr(type(data)) + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) if not data: return @@ -753,7 +767,7 @@ self._loop.add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. - self._buffer.append(data) + self._buffer.extend(data) self._maybe_pause_protocol() def can_write_eof(self): @@ -762,6 +776,8 @@ class _SelectorDatagramTransport(_SelectorTransport): + _buffer_factory = collections.deque + def __init__(self, loop, sock, protocol, address=None, extra=None): super().__init__(loop, sock, protocol, extra) self._address = address @@ -784,12 +800,15 @@ self._protocol.datagram_received(data, addr) def sendto(self, data, addr=None): - assert isinstance(data, bytes), repr(type(data)) + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) if not data: return - if self._address: - assert addr in (None, self._address) + if self._address and addr not in (None, self._address): + raise ValueError('Invalid address: must be None or %s' % + (self._address,)) if self._conn_lost and self._address: if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: @@ -814,7 +833,8 @@ self._fatal_error(exc) return - self._buffer.append((data, addr)) + # Ensure that what we buffer is immutable. + self._buffer.append((bytes(data), addr)) self._maybe_pause_protocol() def _sendto_ready(self): diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -32,6 +32,10 @@ self._internal_fds += 1 +def list_to_buffer(l=()): + return bytearray().join(l) + + class BaseSelectorEventLoopTests(unittest.TestCase): def setUp(self): @@ -613,7 +617,7 @@ def test_close_write_buffer(self): tr = _SelectorTransport(self.loop, self.sock, self.protocol, None) - tr._buffer.append(b'data') + tr._buffer.extend(b'data') tr.close() self.assertFalse(self.loop.readers) @@ -622,13 +626,13 @@ def test_force_close(self): tr = _SelectorTransport(self.loop, self.sock, self.protocol, None) - tr._buffer.append(b'1') + tr._buffer.extend(b'1') self.loop.add_reader(7, unittest.mock.sentinel) self.loop.add_writer(7, unittest.mock.sentinel) tr._force_close(None) self.assertTrue(tr._closing) - self.assertEqual(tr._buffer, collections.deque()) + self.assertEqual(tr._buffer, list_to_buffer()) self.assertFalse(self.loop.readers) self.assertFalse(self.loop.writers) @@ -783,21 +787,40 @@ transport.write(data) self.sock.send.assert_called_with(data) + def test_write_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = len(data) + + transport = _SelectorSocketTransport( + self.loop, self.sock, self.protocol) + transport.write(data) + self.sock.send.assert_called_with(data) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + + transport = _SelectorSocketTransport( + self.loop, self.sock, self.protocol) + transport.write(data) + self.sock.send.assert_called_with(data) + def test_write_no_data(self): transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer.append(b'data') + transport._buffer.extend(b'data') transport.write(b'') self.assertFalse(self.sock.send.called) - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) def test_write_buffer(self): transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer.append(b'data1') + transport._buffer.extend(b'data1') transport.write(b'data2') self.assertFalse(self.sock.send.called) - self.assertEqual(collections.deque([b'data1', b'data2']), + self.assertEqual(list_to_buffer([b'data1', b'data2']), transport._buffer) def test_write_partial(self): @@ -809,7 +832,30 @@ transport.write(data) self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'ta']), transport._buffer) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_partial_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = 2 + + transport = _SelectorSocketTransport( + self.loop, self.sock, self.protocol) + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_partial_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + + transport = _SelectorSocketTransport( + self.loop, self.sock, self.protocol) + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) def test_write_partial_none(self): data = b'data' @@ -821,7 +867,7 @@ transport.write(data) self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) def test_write_tryagain(self): self.sock.send.side_effect = BlockingIOError @@ -832,7 +878,7 @@ transport.write(data) self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) @unittest.mock.patch('asyncio.selector_events.logger') def test_write_exception(self, m_log): @@ -859,7 +905,7 @@ def test_write_str(self): transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - self.assertRaises(AssertionError, transport.write, 'str') + self.assertRaises(TypeError, transport.write, 'str') def test_write_closing(self): transport = _SelectorSocketTransport( @@ -875,11 +921,10 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer.append(data) + transport._buffer.extend(data) self.loop.add_writer(7, transport._write_ready) transport._write_ready() self.assertTrue(self.sock.send.called) - self.assertEqual(self.sock.send.call_args[0], (data,)) self.assertFalse(self.loop.writers) def test_write_ready_closing(self): @@ -889,10 +934,10 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) transport._closing = True - transport._buffer.append(data) + transport._buffer.extend(data) self.loop.add_writer(7, transport._write_ready) transport._write_ready() - self.sock.send.assert_called_with(data) + self.assertTrue(self.sock.send.called) self.assertFalse(self.loop.writers) self.sock.close.assert_called_with() self.protocol.connection_lost.assert_called_with(None) @@ -900,6 +945,7 @@ def test_write_ready_no_data(self): transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) + # This is an internal error. self.assertRaises(AssertionError, transport._write_ready) def test_write_ready_partial(self): @@ -908,11 +954,11 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer.append(data) + transport._buffer.extend(data) self.loop.add_writer(7, transport._write_ready) transport._write_ready() self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'ta']), transport._buffer) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) def test_write_ready_partial_none(self): data = b'data' @@ -920,23 +966,23 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer.append(data) + transport._buffer.extend(data) self.loop.add_writer(7, transport._write_ready) transport._write_ready() self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) def test_write_ready_tryagain(self): self.sock.send.side_effect = BlockingIOError transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) - transport._buffer = collections.deque([b'data1', b'data2']) + transport._buffer = list_to_buffer([b'data1', b'data2']) self.loop.add_writer(7, transport._write_ready) transport._write_ready() self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(collections.deque([b'data1data2']), transport._buffer) + self.assertEqual(list_to_buffer([b'data1data2']), transport._buffer) def test_write_ready_exception(self): err = self.sock.send.side_effect = OSError() @@ -944,7 +990,7 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) transport._fatal_error = unittest.mock.Mock() - transport._buffer.append(b'data') + transport._buffer.extend(b'data') transport._write_ready() transport._fatal_error.assert_called_with(err) @@ -956,7 +1002,7 @@ transport = _SelectorSocketTransport( self.loop, self.sock, self.protocol) transport.close() - transport._buffer.append(b'data') + transport._buffer.extend(b'data') transport._write_ready() remove_writer.assert_called_with(self.sock_fd) @@ -976,12 +1022,12 @@ self.sock.send.side_effect = BlockingIOError tr.write(b'data') tr.write_eof() - self.assertEqual(tr._buffer, collections.deque([b'data'])) + self.assertEqual(tr._buffer, list_to_buffer([b'data'])) self.assertTrue(tr._eof) self.assertFalse(self.sock.shutdown.called) self.sock.send.side_effect = lambda _: 4 tr._write_ready() - self.sock.send.assert_called_with(b'data') + self.assertTrue(self.sock.send.called) self.sock.shutdown.assert_called_with(socket.SHUT_WR) tr.close() @@ -1065,15 +1111,34 @@ self.assertFalse(tr._paused) self.loop.assert_reader(1, tr._read_ready) + def test_write(self): + transport = self._make_one() + transport.write(b'data') + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_bytearray(self): + transport = self._make_one() + data = bytearray(b'data') + transport.write(data) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + self.assertIsNot(data, transport._buffer) # Hasn't been incorporated. + + def test_write_memoryview(self): + transport = self._make_one() + data = memoryview(b'data') + transport.write(data) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + def test_write_no_data(self): transport = self._make_one() - transport._buffer.append(b'data') + transport._buffer.extend(b'data') transport.write(b'') - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) def test_write_str(self): transport = self._make_one() - self.assertRaises(AssertionError, transport.write, 'str') + self.assertRaises(TypeError, transport.write, 'str') def test_write_closing(self): transport = self._make_one() @@ -1087,7 +1152,7 @@ transport = self._make_one() transport._conn_lost = 1 transport.write(b'data') - self.assertEqual(transport._buffer, collections.deque()) + self.assertEqual(transport._buffer, list_to_buffer()) transport.write(b'data') transport.write(b'data') transport.write(b'data') @@ -1107,7 +1172,7 @@ transport = self._make_one() transport._write_wants_read = True transport._write_ready = unittest.mock.Mock() - transport._buffer.append(b'data') + transport._buffer.extend(b'data') transport._read_ready() self.assertFalse(transport._write_wants_read) @@ -1168,31 +1233,31 @@ def test_write_ready_send(self): self.sslsock.send.return_value = 4 transport = self._make_one() - transport._buffer = collections.deque([b'data']) + transport._buffer = list_to_buffer([b'data']) transport._write_ready() - self.assertEqual(collections.deque(), transport._buffer) + self.assertEqual(list_to_buffer(), transport._buffer) self.assertTrue(self.sslsock.send.called) def test_write_ready_send_none(self): self.sslsock.send.return_value = 0 transport = self._make_one() - transport._buffer = collections.deque([b'data1', b'data2']) + transport._buffer = list_to_buffer([b'data1', b'data2']) transport._write_ready() self.assertTrue(self.sslsock.send.called) - self.assertEqual(collections.deque([b'data1data2']), transport._buffer) + self.assertEqual(list_to_buffer([b'data1data2']), transport._buffer) def test_write_ready_send_partial(self): self.sslsock.send.return_value = 2 transport = self._make_one() - transport._buffer = collections.deque([b'data1', b'data2']) + transport._buffer = list_to_buffer([b'data1', b'data2']) transport._write_ready() self.assertTrue(self.sslsock.send.called) - self.assertEqual(collections.deque([b'ta1data2']), transport._buffer) + self.assertEqual(list_to_buffer([b'ta1data2']), transport._buffer) def test_write_ready_send_closing_partial(self): self.sslsock.send.return_value = 2 transport = self._make_one() - transport._buffer = collections.deque([b'data1', b'data2']) + transport._buffer = list_to_buffer([b'data1', b'data2']) transport._write_ready() self.assertTrue(self.sslsock.send.called) self.assertFalse(self.sslsock.close.called) @@ -1201,7 +1266,7 @@ self.sslsock.send.return_value = 4 transport = self._make_one() transport.close() - transport._buffer = collections.deque([b'data']) + transport._buffer = list_to_buffer([b'data']) transport._write_ready() self.assertFalse(self.loop.writers) self.protocol.connection_lost.assert_called_with(None) @@ -1210,26 +1275,26 @@ self.sslsock.send.return_value = 4 transport = self._make_one() transport.close() - transport._buffer = collections.deque() + transport._buffer = list_to_buffer() transport._write_ready() self.assertFalse(self.loop.writers) self.protocol.connection_lost.assert_called_with(None) def test_write_ready_send_retry(self): transport = self._make_one() - transport._buffer = collections.deque([b'data']) + transport._buffer = list_to_buffer([b'data']) self.sslsock.send.side_effect = ssl.SSLWantWriteError transport._write_ready() - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) self.sslsock.send.side_effect = BlockingIOError() transport._write_ready() - self.assertEqual(collections.deque([b'data']), transport._buffer) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) def test_write_ready_send_read(self): transport = self._make_one() - transport._buffer = collections.deque([b'data']) + transport._buffer = list_to_buffer([b'data']) self.loop.remove_writer = unittest.mock.Mock() self.sslsock.send.side_effect = ssl.SSLWantReadError @@ -1242,11 +1307,11 @@ err = self.sslsock.send.side_effect = OSError() transport = self._make_one() - transport._buffer = collections.deque([b'data']) + transport._buffer = list_to_buffer([b'data']) transport._fatal_error = unittest.mock.Mock() transport._write_ready() transport._fatal_error.assert_called_with(err) - self.assertEqual(collections.deque(), transport._buffer) + self.assertEqual(list_to_buffer(), transport._buffer) def test_write_ready_read_wants_write(self): self.loop.add_reader = unittest.mock.Mock() @@ -1355,6 +1420,24 @@ self.assertEqual( self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + def test_sendto_bytearray(self): + data = bytearray(b'data') + transport = _SelectorDatagramTransport( + self.loop, self.sock, self.protocol) + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_memoryview(self): + data = memoryview(b'data') + transport = _SelectorDatagramTransport( + self.loop, self.sock, self.protocol) + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + def test_sendto_no_data(self): transport = _SelectorDatagramTransport( self.loop, self.sock, self.protocol) @@ -1375,6 +1458,32 @@ (b'data2', ('0.0.0.0', 12345))], list(transport._buffer)) + def test_sendto_buffer_bytearray(self): + data2 = bytearray(b'data2') + transport = _SelectorDatagramTransport( + self.loop, self.sock, self.protocol) + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_memoryview(self): + data2 = memoryview(b'data2') + transport = _SelectorDatagramTransport( + self.loop, self.sock, self.protocol) + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_tryagain(self): data = b'data' @@ -1439,13 +1548,13 @@ def test_sendto_str(self): transport = _SelectorDatagramTransport( self.loop, self.sock, self.protocol) - self.assertRaises(AssertionError, transport.sendto, 'str', ()) + self.assertRaises(TypeError, transport.sendto, 'str', ()) def test_sendto_connected_addr(self): transport = _SelectorDatagramTransport( self.loop, self.sock, self.protocol, ('0.0.0.0', 1)) self.assertRaises( - AssertionError, transport.sendto, b'str', ('0.0.0.0', 2)) + ValueError, transport.sendto, b'str', ('0.0.0.0', 2)) def test_sendto_closing(self): transport = _SelectorDatagramTransport( -- Repository URL: http://hg.python.org/cpython

2 1

cpython: Change pathlib documentation to use "raise" instead of "throw".
by ned.deily 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/acabd3f035fe changeset: 87619:acabd3f035fe user: Ned Deily <nad(a)acm.org> date: Wed Nov 27 14:42:55 2013 -0800 summary: Change pathlib documentation to use "raise" instead of "throw". files: Doc/library/pathlib.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -655,7 +655,7 @@ .. method:: Path.group() - Return the name of the group owning the file. :exc:`KeyError` is thrown + Return the name of the group owning the file. :exc:`KeyError` is raised if the file's gid isn't found in the system database. @@ -774,7 +774,7 @@ .. method:: Path.owner() - Return the name of the user owning the file. :exc:`KeyError` is thrown + Return the name of the user owning the file. :exc:`KeyError` is raised if the file's uid isn't found in the system database. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Issue #19818: tracemalloc, the number of frame limit cannot be zero anymore
by victor.stinner 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/8df54b9b99ef changeset: 87618:8df54b9b99ef user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Nov 27 23:39:55 2013 +0100 summary: Issue #19818: tracemalloc, the number of frame limit cannot be zero anymore files: Doc/library/tracemalloc.rst | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -391,9 +391,9 @@ If *all_frames* is ``True``, all frames of the traceback are checked. If *all_frames* is ``False``, only the most recent frame is checked. - This attribute is ignored if the traceback limit is less than ``2``. See - the :func:`get_traceback_limit` function and - :attr:`Snapshot.traceback_limit` attribute. + This attribute has no effect if the traceback limit is ``1``. See the + :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit` + attribute. Frame -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Closes #19786: tracemalloc, remove the arbitrary limit of 100 frames
by victor.stinner 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/eead17ba32d8 changeset: 87616:eead17ba32d8 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Nov 27 22:27:13 2013 +0100 summary: Closes #19786: tracemalloc, remove the arbitrary limit of 100 frames The limit is now 178,956,969 on 64 bit (it is greater on 32 bit because structures are smaller). Use int instead of Py_ssize_t to store the number of frames to have smaller traceback_t objects. files: Lib/test/test_tracemalloc.py | 12 ++-- Modules/_tracemalloc.c | 55 ++++++++++++++--------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -747,14 +747,14 @@ self.assertEqual(stdout, b'10') def test_env_var_invalid(self): - for nframe in (-1, 0, 5000): + for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): with support.SuppressCrashReport(): ok, stdout, stderr = assert_python_failure( '-c', 'pass', PYTHONTRACEMALLOC=str(nframe)) - self.assertIn(b'PYTHONTRACEMALLOC must be an integer ' - b'in range [1; 100]', + self.assertIn(b'PYTHONTRACEMALLOC: invalid ' + b'number of frames', stderr) def test_sys_xoptions(self): @@ -770,13 +770,13 @@ self.assertEqual(stdout, str(nframe).encode('ascii')) def test_sys_xoptions_invalid(self): - for nframe in (-1, 0, 5000): + for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): with support.SuppressCrashReport(): args = ('-X', 'tracemalloc=%s' % nframe, '-c', 'pass') ok, stdout, stderr = assert_python_failure(*args) - self.assertIn(b'-X tracemalloc=NFRAME: number of frame must ' - b'be an integer in range [1; 100]', + self.assertIn(b'-X tracemalloc=NFRAME: invalid ' + b'number of frames', stderr) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -27,11 +27,6 @@ PyMemAllocator obj; } allocators; -/* Arbitrary limit of the number of frames in a traceback. The value was chosen - to not allocate too much memory on the stack (see TRACEBACK_STACK_SIZE - below). */ -#define MAX_NFRAME 100 - static struct { /* Module initialized? Variable protected by the GIL */ @@ -88,7 +83,9 @@ #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) -#define TRACEBACK_STACK_SIZE TRACEBACK_SIZE(MAX_NFRAME) + +#define MAX_NFRAME \ + ((INT_MAX - sizeof(traceback_t)) / sizeof(frame_t) + 1) static PyObject *unknown_filename = NULL; static traceback_t tracemalloc_empty_traceback; @@ -115,6 +112,10 @@ Protected by the GIL */ static _Py_hashtable_t *tracemalloc_filenames = NULL; +/* Buffer to store a new traceback in traceback_new(). + Protected by the GIL. */ +static traceback_t *tracemalloc_traceback = NULL; + /* Hash table used as a set to intern tracebacks: traceback_t* => traceback_t* Protected by the GIL */ @@ -394,8 +395,7 @@ static traceback_t * traceback_new(void) { - char stack_buffer[TRACEBACK_STACK_SIZE]; - traceback_t *traceback = (traceback_t *)stack_buffer; + traceback_t *traceback; _Py_hashtable_entry_t *entry; #ifdef WITH_THREAD @@ -403,6 +403,7 @@ #endif /* get frames */ + traceback = tracemalloc_traceback; traceback->nframe = 0; traceback_get_frames(traceback); if (traceback->nframe == 0) @@ -788,9 +789,10 @@ } static int -tracemalloc_start(void) +tracemalloc_start(int max_nframe) { PyMemAllocator alloc; + size_t size; if (tracemalloc_init() < 0) return -1; @@ -803,6 +805,18 @@ if (tracemalloc_atexit_register() < 0) return -1; + assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); + tracemalloc_config.max_nframe = max_nframe; + + /* allocate a buffer to store a new traceback */ + size = TRACEBACK_SIZE(max_nframe); + assert(tracemalloc_traceback == NULL); + tracemalloc_traceback = raw_malloc(size); + if (tracemalloc_traceback == NULL) { + PyErr_NoMemory(); + return -1; + } + #ifdef TRACE_RAW_MALLOC alloc.malloc = tracemalloc_raw_malloc; alloc.realloc = tracemalloc_raw_realloc; @@ -854,9 +868,10 @@ /* release memory */ tracemalloc_clear_traces(); + raw_free(tracemalloc_traceback); + tracemalloc_traceback = NULL; } - static PyObject* lineno_as_obj(int lineno) { @@ -1194,6 +1209,7 @@ py_tracemalloc_start(PyObject *self, PyObject *args) { Py_ssize_t nframe = 1; + int nframe_int; if (!PyArg_ParseTuple(args, "|n:start", &nframe)) return NULL; @@ -1201,12 +1217,12 @@ if (nframe < 1 || nframe > MAX_NFRAME) { PyErr_Format(PyExc_ValueError, "the number of frames must be in range [1; %i]", - MAX_NFRAME); + (int)MAX_NFRAME); return NULL; } - tracemalloc_config.max_nframe = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); + nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - if (tracemalloc_start() < 0) + if (tracemalloc_start(nframe_int) < 0) return NULL; Py_RETURN_NONE; @@ -1378,16 +1394,15 @@ if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { char *endptr = p; - unsigned long value; + long value; - value = strtoul(p, &endptr, 10); + value = strtol(p, &endptr, 10); if (*endptr != '\0' || value < 1 || value > MAX_NFRAME || (errno == ERANGE && value == ULONG_MAX)) { - Py_FatalError("PYTHONTRACEMALLOC must be an integer " - "in range [1; " STR(MAX_NFRAME) "]"); + Py_FatalError("PYTHONTRACEMALLOC: invalid number of frames"); return -1; } @@ -1417,12 +1432,10 @@ nframe = parse_sys_xoptions(value); Py_DECREF(value); if (nframe < 0) { - Py_FatalError("-X tracemalloc=NFRAME: number of frame must be " - "an integer in range [1; " STR(MAX_NFRAME) "]"); + Py_FatalError("-X tracemalloc=NFRAME: invalid number of frames"); } } - tracemalloc_config.max_nframe = nframe; - return tracemalloc_start(); + return tracemalloc_start(nframe); } -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Close #19798: replace "maximum" term with "peak" in get_traced_memory()
by victor.stinner 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/553144bd7bf1 changeset: 87615:553144bd7bf1 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Nov 27 21:39:49 2013 +0100 summary: Close #19798: replace "maximum" term with "peak" in get_traced_memory() documentation. Use also the term "current" for the current size. files: Doc/library/tracemalloc.rst | 4 ++-- Lib/test/test_tracemalloc.py | 14 +++++++------- Modules/_tracemalloc.c | 24 ++++++++++++------------ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -276,8 +276,8 @@ .. function:: get_traced_memory() - Get the current size and maximum size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(size: int, max_size: int)``. + Get the current size and peak size of memory blocks traced by the + :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. .. function:: get_tracemalloc_memory() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -182,19 +182,19 @@ obj_size = 1024 * 1024 tracemalloc.clear_traces() obj, obj_traceback = allocate_bytes(obj_size) - size, max_size = tracemalloc.get_traced_memory() + size, peak_size = tracemalloc.get_traced_memory() self.assertGreaterEqual(size, obj_size) - self.assertGreaterEqual(max_size, size) + self.assertGreaterEqual(peak_size, size) self.assertLessEqual(size - obj_size, max_error) - self.assertLessEqual(max_size - size, max_error) + self.assertLessEqual(peak_size - size, max_error) # destroy the object obj = None - size2, max_size2 = tracemalloc.get_traced_memory() + size2, peak_size2 = tracemalloc.get_traced_memory() self.assertLess(size2, size) self.assertGreaterEqual(size - size2, obj_size - max_error) - self.assertGreaterEqual(max_size2, max_size) + self.assertGreaterEqual(peak_size2, peak_size) # clear_traces() must reset traced memory counters tracemalloc.clear_traces() @@ -202,8 +202,8 @@ # allocate another object obj, obj_traceback = allocate_bytes(obj_size) - size, max_size = tracemalloc.get_traced_memory() - self.assertGreater(size, 0) + size, peak_size = tracemalloc.get_traced_memory() + self.assertGreaterEqual(size, obj_size) # stop() also resets traced memory counters tracemalloc.stop() diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -106,9 +106,9 @@ Protected by TABLES_LOCK(). */ static size_t tracemalloc_traced_memory = 0; -/* Maximum size in bytes of traced memory. +/* Peak size in bytes of traced memory. Protected by TABLES_LOCK(). */ -static size_t tracemalloc_max_traced_memory = 0; +static size_t tracemalloc_peak_traced_memory = 0; /* Hash table used as a set to to intern filenames: PyObject* => PyObject*. @@ -464,8 +464,8 @@ if (res == 0) { assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); tracemalloc_traced_memory += size; - if (tracemalloc_traced_memory > tracemalloc_max_traced_memory) - tracemalloc_max_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_traced_memory > tracemalloc_peak_traced_memory) + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; } TABLES_UNLOCK(); @@ -674,7 +674,7 @@ TABLES_LOCK(); _Py_hashtable_clear(tracemalloc_traces); tracemalloc_traced_memory = 0; - tracemalloc_max_traced_memory = 0; + tracemalloc_peak_traced_memory = 0; TABLES_UNLOCK(); _Py_hashtable_foreach(tracemalloc_tracebacks, traceback_free_traceback, NULL); @@ -1266,26 +1266,26 @@ PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, "get_traced_memory() -> (int, int)\n" "\n" - "Get the current size and maximum size of memory blocks traced\n" - "by the tracemalloc module as a tuple: (size: int, max_size: int)."); + "Get the current size and peak size of memory blocks traced\n" + "by the tracemalloc module as a tuple: (current: int, peak: int)."); static PyObject* tracemalloc_get_traced_memory(PyObject *self) { - Py_ssize_t size, max_size; - PyObject *size_obj, *max_size_obj; + Py_ssize_t size, peak_size; + PyObject *size_obj, *peak_size_obj; if (!tracemalloc_config.tracing) return Py_BuildValue("ii", 0, 0); TABLES_LOCK(); size = tracemalloc_traced_memory; - max_size = tracemalloc_max_traced_memory; + peak_size = tracemalloc_peak_traced_memory; TABLES_UNLOCK(); size_obj = PyLong_FromSize_t(size); - max_size_obj = PyLong_FromSize_t(max_size); - return Py_BuildValue("NN", size_obj, max_size_obj); + peak_size_obj = PyLong_FromSize_t(peak_size); + return Py_BuildValue("NN", size_obj, peak_size_obj); } static PyMethodDef module_methods[] = { -- Repository URL: http://hg.python.org/cpython

1 0

cpython: asyncio: Fix get_event_loop() to call set_event_loop() when setting the loop.
by guido.van.rossum 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/5c9af8194d3b changeset: 87614:5c9af8194d3b user: Guido van Rossum <guido(a)python.org> date: Wed Nov 27 10:37:13 2013 -0800 summary: asyncio: Fix get_event_loop() to call set_event_loop() when setting the loop. By Anthony Baire. files: Lib/asyncio/events.py | 2 +- Lib/test/test_asyncio/test_events.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletions(-) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -360,7 +360,7 @@ if (self._local._loop is None and not self._local._set_called and isinstance(threading.current_thread(), threading._MainThread)): - self._local._loop = self.new_event_loop() + self.set_event_loop(self.new_event_loop()) assert self._local._loop is not None, \ ('There is no current event loop in thread %r.' % threading.current_thread().name) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1599,6 +1599,22 @@ self.assertIs(loop, policy.get_event_loop()) loop.close() + def test_get_event_loop_calls_set_event_loop(self): + policy = self.create_policy() + + with unittest.mock.patch.object( + policy, "set_event_loop", + wraps=policy.set_event_loop) as m_set_event_loop: + + loop = policy.get_event_loop() + + # policy._local._loop must be set through .set_event_loop() + # (the unix DefaultEventLoopPolicy needs this call to attach + # the child watcher correctly) + m_set_event_loop.assert_called_with(loop) + + loop.close() + def test_get_event_loop_after_set_none(self): policy = self.create_policy() policy.set_event_loop(None) -- Repository URL: http://hg.python.org/cpython

1 0

peps: Remove question to myself about set_child_watcher() now the issue is fixed.
by guido.van.rossum 27 Nov '13

27 Nov '13

http://hg.python.org/peps/rev/ee7c8a3b3b6a changeset: 5324:ee7c8a3b3b6a user: Guido van Rossum <guido(a)dropbox.com> date: Wed Nov 27 10:22:42 2013 -0800 summary: Remove question to myself about set_child_watcher() now the issue is fixed. files: pep-3156.txt | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/pep-3156.txt b/pep-3156.txt --- a/pep-3156.txt +++ b/pep-3156.txt @@ -1863,7 +1863,6 @@ watcher = asyncio.FastChildWatcher() asyncio.set_child_watcher(watcher) - # TBD: Is a call to watcher.attach_loop() needed? Wish List -- Repository URL: http://hg.python.org/peps

1 0

cpython: add SO_PRIORITY (closes #19802)
by benjamin.peterson 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/9bbada125b3f changeset: 87613:9bbada125b3f user: Benjamin Peterson <benjamin(a)python.org> date: Wed Nov 27 09:18:54 2013 -0600 summary: add SO_PRIORITY (closes #19802) Patch by Claudiu Popa. files: Misc/NEWS | 2 ++ Modules/socketmodule.c | 3 +++ 2 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,8 @@ Library ------- +- Issue #19802: Add socket.SO_PRIORITY. + - Issue #11508: Fixed uuid.getnode() and uuid.uuid1() on environment with virtual interface. Original patch by Kent Frazier. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6241,6 +6241,9 @@ #ifdef SO_BINDTODEVICE PyModule_AddIntMacro(m, SO_BINDTODEVICE); #endif +#ifdef SO_PRIORITY + PyModule_AddIntMacro(m, SO_PRIORITY); +#endif /* Maximum number of connections for "listen" */ #ifdef SOMAXCONN -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Combine the FastCall functions in cpickle.
by alexandre.vassalotti 27 Nov '13

27 Nov '13

http://hg.python.org/cpython/rev/23459df0753e changeset: 87612:23459df0753e user: Alexandre Vassalotti <alexandre(a)peadrop.com> date: Wed Nov 27 02:26:54 2013 -0800 summary: Combine the FastCall functions in cpickle. I fixed the bug that was in my previous attempt of this cleanup. I ran the full test suite to verify I didn't introduce any obvious bugs. files: Modules/_pickle.c | 136 +++++++++++----------------------- 1 files changed, 44 insertions(+), 92 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -346,7 +346,6 @@ pickling. */ PyObject *pers_func; /* persistent_id() method, can be NULL */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ - PyObject *arg; PyObject *write; /* write() method of the output stream. */ PyObject *output_buffer; /* Write into a local bytearray buffer before @@ -383,7 +382,6 @@ Py_ssize_t memo_size; /* Capacity of the memo array */ Py_ssize_t memo_len; /* Number of objects in the memo */ - PyObject *arg; PyObject *pers_func; /* persistent_load() method, can be NULL. */ Py_buffer buffer; @@ -639,57 +637,37 @@ /*************************************************************************/ -/* Helpers for creating the argument tuple passed to functions. This has the - performance advantage of calling PyTuple_New() only once. - - XXX(avassalotti): Inline directly in _Pickler_FastCall() and - _Unpickler_FastCall(). */ -#define ARG_TUP(self, obj) do { \ - if ((self)->arg || ((self)->arg=PyTuple_New(1))) { \ - Py_XDECREF(PyTuple_GET_ITEM((self)->arg, 0)); \ - PyTuple_SET_ITEM((self)->arg, 0, (obj)); \ - } \ - else { \ - Py_DECREF((obj)); \ - } \ - } while (0) - -#define FREE_ARG_TUP(self) do { \ - if ((self)->arg->ob_refcnt > 1) \ - Py_CLEAR((self)->arg); \ - } while (0) - -/* A temporary cleaner API for fast single argument function call. - - XXX: Does caching the argument tuple provides any real performance benefits? - - A quick benchmark, on a 2.0GHz Athlon64 3200+ running Linux 2.6.24 with - glibc 2.7, tells me that it takes roughly 20,000,000 PyTuple_New(1) calls - when the tuple is retrieved from the freelist (i.e, call PyTuple_New() then - immediately DECREF it) and 1,200,000 calls when allocating brand new tuples - (i.e, call PyTuple_New() and store the returned value in an array), to save - one second (wall clock time). Either ways, the loading time a pickle stream - large enough to generate this number of calls would be massively - overwhelmed by other factors, like I/O throughput, the GC traversal and - object allocation overhead. So, I really doubt these functions provide any - real benefits. - - On the other hand, oprofile reports that pickle spends a lot of time in - these functions. But, that is probably more related to the function call - overhead, than the argument tuple allocation. - - XXX: And, what is the reference behavior of these? Steal, borrow? At first - glance, it seems to steal the reference of 'arg' and borrow the reference - of 'func'. */ +/* Helper for calling a function with a single argument quickly. + + This has the performance advantage of reusing the argument tuple. This + provides a nice performance boost with older pickle protocols where many + unbuffered reads occurs. + + This function steals the reference of the given argument. */ static PyObject * -_Pickler_FastCall(PicklerObject *self, PyObject *func, PyObject *arg) -{ - PyObject *result = NULL; - - ARG_TUP(self, arg); - if (self->arg) { - result = PyObject_Call(func, self->arg, NULL); - FREE_ARG_TUP(self); +_Pickle_FastCall(PyObject *func, PyObject *obj) +{ + static PyObject *arg_tuple = NULL; + PyObject *result; + + if (arg_tuple == NULL) { + arg_tuple = PyTuple_New(1); + if (arg_tuple == NULL) { + Py_DECREF(obj); + return NULL; + } + } + assert(arg_tuple->ob_refcnt == 1); + assert(PyTuple_GET_ITEM(arg_tuple, 0) == NULL); + + PyTuple_SET_ITEM(arg_tuple, 0, obj); + result = PyObject_Call(func, arg_tuple, NULL); + + Py_CLEAR(PyTuple_GET_ITEM(arg_tuple, 0)); + if (arg_tuple->ob_refcnt > 1) { + /* The function called saved a reference to our argument tuple. + This means we cannot reuse it anymore. */ + Py_CLEAR(arg_tuple); } return result; } @@ -787,7 +765,7 @@ if (output == NULL) return -1; - result = _Pickler_FastCall(self, self->write, output); + result = _Pickle_FastCall(self->write, output); Py_XDECREF(result); return (result == NULL) ? -1 : 0; } @@ -853,7 +831,6 @@ self->pers_func = NULL; self->dispatch_table = NULL; - self->arg = NULL; self->write = NULL; self->proto = 0; self->bin = 0; @@ -922,20 +899,6 @@ return 0; } -/* See documentation for _Pickler_FastCall(). */ -static PyObject * -_Unpickler_FastCall(UnpicklerObject *self, PyObject *func, PyObject *arg) -{ - PyObject *result = NULL; - - ARG_TUP(self, arg); - if (self->arg) { - result = PyObject_Call(func, self->arg, NULL); - FREE_ARG_TUP(self); - } - return result; -} - /* Returns the size of the input on success, -1 on failure. This takes its own reference to `input`. */ static Py_ssize_t @@ -1006,7 +969,7 @@ PyObject *len = PyLong_FromSsize_t(n); if (len == NULL) return -1; - data = _Unpickler_FastCall(self, self->read, len); + data = _Pickle_FastCall(self->read, len); } if (data == NULL) return -1; @@ -1019,7 +982,7 @@ Py_DECREF(data); return -1; } - prefetched = _Unpickler_FastCall(self, self->peek, len); + prefetched = _Pickle_FastCall(self->peek, len); if (prefetched == NULL) { if (PyErr_ExceptionMatches(PyExc_NotImplementedError)) { /* peek() is probably not supported by the given file object */ @@ -1229,7 +1192,6 @@ if (self == NULL) return NULL; - self->arg = NULL; self->pers_func = NULL; self->input_buffer = NULL; self->input_line = NULL; @@ -3172,7 +3134,7 @@ const char binpersid_op = BINPERSID; Py_INCREF(obj); - pid = _Pickler_FastCall(self, func, obj); + pid = _Pickle_FastCall(func, obj); if (pid == NULL) return -1; @@ -3600,7 +3562,7 @@ } if (reduce_func != NULL) { Py_INCREF(obj); - reduce_value = _Pickler_FastCall(self, reduce_func, obj); + reduce_value = _Pickle_FastCall(reduce_func, obj); } else if (PyType_IsSubtype(type, &PyType_Type)) { status = save_global(self, obj, NULL); @@ -3625,7 +3587,7 @@ PyObject *proto; proto = PyLong_FromLong(self->proto); if (proto != NULL) { - reduce_value = _Pickler_FastCall(self, reduce_func, proto); + reduce_value = _Pickle_FastCall(reduce_func, proto); } } else { @@ -3794,7 +3756,6 @@ Py_XDECREF(self->write); Py_XDECREF(self->pers_func); Py_XDECREF(self->dispatch_table); - Py_XDECREF(self->arg); Py_XDECREF(self->fast_memo); PyMemoTable_Del(self->memo); @@ -3808,7 +3769,6 @@ Py_VISIT(self->write); Py_VISIT(self->pers_func); Py_VISIT(self->dispatch_table); - Py_VISIT(self->arg); Py_VISIT(self->fast_memo); return 0; } @@ -3820,7 +3780,6 @@ Py_CLEAR(self->write); Py_CLEAR(self->pers_func); Py_CLEAR(self->dispatch_table); - Py_CLEAR(self->arg); Py_CLEAR(self->fast_memo); if (self->memo != NULL) { @@ -3943,7 +3902,6 @@ return NULL; } - self->arg = NULL; self->fast = 0; self->fast_nesting = 0; self->fast_memo = NULL; @@ -5208,9 +5166,9 @@ if (pid == NULL) return -1; - /* Ugh... this does not leak since _Unpickler_FastCall() steals the - reference to pid first. */ - pid = _Unpickler_FastCall(self, self->pers_func, pid); + /* This does not leak since _Pickle_FastCall() steals the reference + to pid first. */ + pid = _Pickle_FastCall(self->pers_func, pid); if (pid == NULL) return -1; @@ -5235,9 +5193,9 @@ if (pid == NULL) return -1; - /* Ugh... this does not leak since _Unpickler_FastCall() steals the + /* This does not leak since _Pickle_FastCall() steals the reference to pid first. */ - pid = _Unpickler_FastCall(self, self->pers_func, pid); + pid = _Pickle_FastCall(self->pers_func, pid); if (pid == NULL) return -1; @@ -5585,7 +5543,7 @@ PyObject *result; value = self->stack->data[i]; - result = _Unpickler_FastCall(self, append_func, value); + result = _Pickle_FastCall(append_func, value); if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SIZE(self->stack) = x; @@ -5700,7 +5658,7 @@ PyObject *item; item = self->stack->data[i]; - result = _Unpickler_FastCall(self, add_func, item); + result = _Pickle_FastCall(add_func, item); if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SIZE(self->stack) = mark; @@ -5747,9 +5705,7 @@ PyObject *result; /* The explicit __setstate__ is responsible for everything. */ - /* Ugh... this does not leak since _Unpickler_FastCall() steals the - reference to state first. */ - result = _Unpickler_FastCall(self, setstate, state); + result = _Pickle_FastCall(setstate, state); Py_DECREF(setstate); if (result == NULL) return -1; @@ -6249,7 +6205,6 @@ Py_XDECREF(self->peek); Py_XDECREF(self->stack); Py_XDECREF(self->pers_func); - Py_XDECREF(self->arg); if (self->buffer.buf != NULL) { PyBuffer_Release(&self->buffer); self->buffer.buf = NULL; @@ -6272,7 +6227,6 @@ Py_VISIT(self->peek); Py_VISIT(self->stack); Py_VISIT(self->pers_func); - Py_VISIT(self->arg); return 0; } @@ -6284,7 +6238,6 @@ Py_CLEAR(self->peek); Py_CLEAR(self->stack); Py_CLEAR(self->pers_func); - Py_CLEAR(self->arg); if (self->buffer.buf != NULL) { PyBuffer_Release(&self->buffer); self->buffer.buf = NULL; @@ -6423,7 +6376,6 @@ if (self->memo == NULL) return NULL; - self->arg = NULL; self->proto = 0; return Py_None; -- Repository URL: http://hg.python.org/cpython

1 0