Mailman 3 November 2015 - Python-checkins

cpython (merge 3.5 -> default): Issue #25395: Fixed crash when highly nested OrderedDict structures were
by serhiy.storchaka Nov. 1, 2015

Nov. 1, 2015

https://hg.python.org/cpython/rev/bd6bfa5fe203 changeset: 98923:bd6bfa5fe203 parent: 98921:f7db966c9fee parent: 98922:ce8c850a52d4 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 01 16:13:45 2015 +0200 summary: Issue #25395: Fixed crash when highly nested OrderedDict structures were garbage collected. files: Lib/test/test_collections.py | 24 ++++++++++++++++++++++++ Misc/NEWS | 3 +++ Objects/odictobject.c | 17 ++++++++++++++--- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -2033,6 +2033,30 @@ items = [('a', 1), ('c', 3), ('b', 2)] self.assertEqual(list(MyOD(items).items()), items) + def test_highly_nested(self): + # Issue 25395: crashes during garbage collection + OrderedDict = self.module.OrderedDict + obj = None + for _ in range(1000): + obj = OrderedDict([(None, obj)]) + del obj + support.gc_collect() + + def test_highly_nested_subclass(self): + # Issue 25395: crashes during garbage collection + OrderedDict = self.module.OrderedDict + deleted = [] + class MyOD(OrderedDict): + def __del__(self): + deleted.append(self.i) + obj = None + for i in range(100): + obj = MyOD([(None, obj)]) + obj.i = i + del obj + support.gc_collect() + self.assertEqual(deleted, list(reversed(range(100)))) + class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #25395: Fixed crash when highly nested OrderedDict structures were + garbage collected. + - Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now between 2x and 3.5x faster. diff --git a/Objects/odictobject.c b/Objects/odictobject.c --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1431,17 +1431,28 @@ static void odict_dealloc(PyODictObject *self) { + PyThreadState *tstate = PyThreadState_GET(); + PyObject_GC_UnTrack(self); - Py_TRASHCAN_SAFE_BEGIN(self); + Py_TRASHCAN_SAFE_BEGIN(self) + Py_XDECREF(self->od_inst_dict); if (self->od_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); _odict_clear_nodes(self); - Py_TRASHCAN_SAFE_END(self); - /* must be last */ + /* Call the base tp_dealloc(). Since it too uses the trashcan mechanism, + * temporarily decrement trash_delete_nesting to prevent triggering it + * and putting the partially deallocated object on the trashcan's + * to-be-deleted-later list. + */ + --tstate->trash_delete_nesting; + assert(_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL); PyDict_Type.tp_dealloc((PyObject *)self); + ++tstate->trash_delete_nesting; + + Py_TRASHCAN_SAFE_END(self) }; /* tp_repr */ -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.5): Issue #25395: Fixed crash when highly nested OrderedDict structures were
by serhiy.storchaka Nov. 1, 2015

Nov. 1, 2015

https://hg.python.org/cpython/rev/ce8c850a52d4 changeset: 98922:ce8c850a52d4 branch: 3.5 parent: 98920:f884590ee620 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Sun Nov 01 16:12:34 2015 +0200 summary: Issue #25395: Fixed crash when highly nested OrderedDict structures were garbage collected. files: Lib/test/test_collections.py | 24 ++++++++++++++++++++++++ Misc/NEWS | 3 +++ Objects/odictobject.c | 17 ++++++++++++++--- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -2025,6 +2025,30 @@ items = [('a', 1), ('c', 3), ('b', 2)] self.assertEqual(list(MyOD(items).items()), items) + def test_highly_nested(self): + # Issue 25395: crashes during garbage collection + OrderedDict = self.module.OrderedDict + obj = None + for _ in range(1000): + obj = OrderedDict([(None, obj)]) + del obj + support.gc_collect() + + def test_highly_nested_subclass(self): + # Issue 25395: crashes during garbage collection + OrderedDict = self.module.OrderedDict + deleted = [] + class MyOD(OrderedDict): + def __del__(self): + deleted.append(self.i) + obj = None + for i in range(100): + obj = MyOD([(None, obj)]) + obj.i = i + del obj + support.gc_collect() + self.assertEqual(deleted, list(reversed(range(100)))) + class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,6 +11,9 @@ Core and Builtins ----------------- +- Issue #25395: Fixed crash when highly nested OrderedDict structures were + garbage collected. + - Issue #25274: sys.setrecursionlimit() now raises a RecursionError if the new recursion limit is too low depending at the current recursion depth. Modify also the "lower-water mark" formula to make it monotonic. This mark is used diff --git a/Objects/odictobject.c b/Objects/odictobject.c --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1431,17 +1431,28 @@ static void odict_dealloc(PyODictObject *self) { + PyThreadState *tstate = PyThreadState_GET(); + PyObject_GC_UnTrack(self); - Py_TRASHCAN_SAFE_BEGIN(self); + Py_TRASHCAN_SAFE_BEGIN(self) + Py_XDECREF(self->od_inst_dict); if (self->od_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)self); _odict_clear_nodes(self); - Py_TRASHCAN_SAFE_END(self); - /* must be last */ + /* Call the base tp_dealloc(). Since it too uses the trashcan mechanism, + * temporarily decrement trash_delete_nesting to prevent triggering it + * and putting the partially deallocated object on the trashcan's + * to-be-deleted-later list. + */ + --tstate->trash_delete_nesting; + assert(_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL); PyDict_Type.tp_dealloc((PyObject *)self); + ++tstate->trash_delete_nesting; + + Py_TRASHCAN_SAFE_END(self) }; /* tp_repr */ -- Repository URL: https://hg.python.org/cpython

1 0

Daily reference leaks (f7db966c9fee): sum=17880
by solipsis＠pitrou.net Nov. 1, 2015

Nov. 1, 2015

results for f7db966c9fee on branch "default" -------------------------------------------- test_capi leaked [1599, 1599, 1599] references, sum=4797 test_capi leaked [387, 389, 389] memory blocks, sum=1165 test_functools leaked [0, 2, 2] memory blocks, sum=4 test_threading leaked [3196, 3196, 3196] references, sum=9588 test_threading leaked [774, 776, 776] memory blocks, sum=2326 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog3Lnth8', '--timeout', '7200']

1 0