Mailman 3 June 2019 - Python-checkins

bpo-29984: Improve 'heapq' test coverage (GH-992)
by Raymond Hettinger 31 May '19

31 May '19

https://github.com/python/cpython/commit/664fe3996f7e05ae351526f02b21504bb0… commit: 664fe3996f7e05ae351526f02b21504bb065bcf8 branch: master author: Rob Day <rkd(a)rkd.me.uk> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-05-31T21:13:57-07:00 summary: bpo-29984: Improve 'heapq' test coverage (GH-992) files: M Lib/heapq.py M Lib/test/test_heapq.py diff --git a/Lib/heapq.py b/Lib/heapq.py index 0e3555cf9118..fabefd87f8bf 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -597,5 +597,5 @@ def nlargest(n, iterable, key=None): if __name__ == "__main__": - import doctest - print(doctest.testmod()) + import doctest # pragma: no cover + print(doctest.testmod()) # pragma: no cover diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index 2f8c648d84a5..6c20b6297dfc 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -2,6 +2,7 @@ import random import unittest +import doctest from test import support from unittest import TestCase, skipUnless @@ -26,6 +27,23 @@ def test_c_functions(self): self.assertEqual(getattr(c_heapq, fname).__module__, '_heapq') +def load_tests(loader, tests, ignore): + # The 'merge' function has examples in its docstring which we should test + # with 'doctest'. + # + # However, doctest can't easily find all docstrings in the module (loading + # it through import_fresh_module seems to confuse it), so we specifically + # create a finder which returns the doctests from the merge method. + + class HeapqMergeDocTestFinder: + def find(self, *args, **kwargs): + dtf = doctest.DocTestFinder() + return dtf.find(py_heapq.merge) + + tests.addTests(doctest.DocTestSuite(py_heapq, + test_finder=HeapqMergeDocTestFinder())) + return tests + class TestHeap: def test_push_pop(self): @@ -135,6 +153,13 @@ def test_heappushpop(self): x = self.module.heappushpop(h, 11) self.assertEqual((h, x), ([11], 10)) + def test_heappop_max(self): + # _heapop_max has an optimization for one-item lists which isn't + # covered in other tests, so test that case explicitly here + h = [3, 2] + self.assertEqual(self.module._heappop_max(h), 3) + self.assertEqual(self.module._heappop_max(h), 2) + def test_heapsort(self): # Exercise everything with repeated heapsort checks for trial in range(100): @@ -168,6 +193,12 @@ def test_merge(self): list(self.module.merge(*seqs, key=key, reverse=reverse))) self.assertEqual(list(self.module.merge()), []) + def test_empty_merges(self): + # Merging two empty lists (with or without a key) should produce + # another empty list. + self.assertEqual(list(self.module.merge([], [])), []) + self.assertEqual(list(self.module.merge([], [], key=lambda: 6)), []) + def test_merge_does_not_suppress_index_error(self): # Issue 19018: Heapq.merge suppresses IndexError from user generator def iterable():

1 0

Improve docstring of list.sort (GH-8516)
by Raymond Hettinger 31 May '19

31 May '19

https://github.com/python/cpython/commit/5c22476c01622f11b7745ee693f8b296a9… commit: 5c22476c01622f11b7745ee693f8b296a9d6a761 branch: master author: Tim Hoffmann <2836374+timhoffm(a)users.noreply.github.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2019-05-31T21:10:02-07:00 summary: Improve docstring of list.sort (GH-8516) files: M Objects/clinic/listobject.c.h M Objects/listobject.c diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 7b8e2d9905f9..57f0a48eb083 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -156,7 +156,15 @@ PyDoc_STRVAR(list_sort__doc__, "sort($self, /, *, key=None, reverse=False)\n" "--\n" "\n" -"Stable sort *IN PLACE*."); +"Sort the list in ascending order and return None.\n" +"\n" +"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" +"order of two equal elements is maintained).\n" +"\n" +"If a key function is given, apply it once to each list item and sort them,\n" +"ascending or descending, according to their function values.\n" +"\n" +"The reverse flag can be set to sort in descending order."); #define LIST_SORT_METHODDEF \ {"sort", (PyCFunction)(void(*)(void))list_sort, METH_FASTCALL|METH_KEYWORDS, list_sort__doc__}, @@ -359,4 +367,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=d1d5078edb7d3cf4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73718c0c33798c62 input=a9049054013a1b77]*/ diff --git a/Objects/listobject.c b/Objects/listobject.c index 233f13dbab0e..f8bf45e5f8cd 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2197,12 +2197,20 @@ list.sort key as keyfunc: object = None reverse: bool(accept={int}) = False -Stable sort *IN PLACE*. +Sort the list in ascending order and return None. + +The sort is in-place (i.e. the list itself is modified) and stable (i.e. the +order of two equal elements is maintained). + +If a key function is given, apply it once to each list item and sort them, +ascending or descending, according to their function values. + +The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=b0fcf743982c5b90]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=cb56cd179a713060]*/ { MergeState ms; Py_ssize_t nremaining;

1 0

bpo-36818: Add PyInterpreterState.runtime field. (gh-13129)
by Eric Snow 31 May '19

31 May '19

https://github.com/python/cpython/commit/396e0a8d9dc65453cb9d53500d0a620602… commit: 396e0a8d9dc65453cb9d53500d0a620602656cfe branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-31T21:16:47-06:00 summary: bpo-36818: Add PyInterpreterState.runtime field. (gh-13129) https://bugs.python.org/issue36818 files: A Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst M Include/cpython/pystate.h M Include/internal/pycore_object.h M Include/internal/pycore_pylifecycle.h M Include/internal/pycore_pystate.h M Modules/_threadmodule.c M Python/ceval.c M Python/import.c M Python/pylifecycle.c M Python/pystate.c M Python/sysmodule.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 94b0809cd4f0..74e7fc96bec9 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -110,9 +110,9 @@ struct _ts { * if the thread holds the last reference to the lock, decref'ing the * lock will delete the lock, and that may trigger arbitrary Python code * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). + * _PyRuntimeState.gilstate.tstate_current is already NULL, so only the + * simplest of C code can be allowed to run (in particular it must not be + * possible to release the GIL). * So instead of holding the lock directly, the tstate holds a weakref to * the lock: that's the value of on_delete_data below. Decref'ing a * weakref is harmless. diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 81548f819198..1c5beb01f458 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -19,9 +19,10 @@ PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); * NB: While the object is tracked by the collector, it must be safe to call the * ob_traverse method. * - * Internal note: _PyRuntime.gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and - * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. + * Internal note: _PyRuntimeState.gc.generation0->_gc_prev doesn't have + * any bit flags because it's not object header. So we don't use + * _PyGCHead_PREV() and _PyGCHead_SET_PREV() for it to avoid unnecessary + * bitwise operations. * * The PyObject_GC_Track() function is the public version of this macro. */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8a692ea16495..e30341710c20 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -39,13 +39,10 @@ extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(void); extern PyStatus _PySys_Create( - _PyRuntimeState *runtime, PyInterpreterState *interp, PyObject **sysmod_p); extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict); -extern int _PySys_InitMain( - _PyRuntimeState *runtime, - PyInterpreterState *interp); +extern int _PySys_InitMain(PyInterpreterState *interp); extern PyStatus _PyImport_Init(PyInterpreterState *interp); extern PyStatus _PyExc_Init(void); extern PyStatus _PyErr_Init(void); @@ -86,10 +83,7 @@ extern void _PyHash_Fini(void); extern int _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); -extern void _PyGILState_Init( - _PyRuntimeState *runtime, - PyInterpreterState *interp, - PyThreadState *tstate); +extern void _PyGILState_Init(PyThreadState *tstate); extern void _PyGILState_Fini(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyGC_DumpShutdownStats(_PyRuntimeState *runtime); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3ab4009770c9..520a74b8a61f 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -19,6 +19,9 @@ extern "C" { #include "pycore_pymem.h" #include "pycore_warnings.h" +// forward +struct pyruntimestate; + /* ceval state */ @@ -68,6 +71,7 @@ struct _is { struct _is *next; struct _ts *tstate_head; + struct pyruntimestate *runtime; int64_t id; int64_t id_refcount; @@ -296,12 +300,8 @@ PyAPI_FUNC(void) _PyRuntime_Finalize(void); /* Other */ -PyAPI_FUNC(void) _PyThreadState_Init( - _PyRuntimeState *runtime, - PyThreadState *tstate); -PyAPI_FUNC(void) _PyThreadState_DeleteExcept( - _PyRuntimeState *runtime, - PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( struct _gilstate_runtime_state *gilstate, diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst new file mode 100644 index 000000000000..bb6c56a628e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst @@ -0,0 +1 @@ +Add PyInterpreterState.runtime (and use it). diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index d5e40ef999e3..099afd86d055 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -996,7 +996,7 @@ t_bootstrap(void *boot_raw) tstate = boot->tstate; tstate->thread_id = PyThread_get_thread_ident(); - _PyThreadState_Init(&_PyRuntime, tstate); + _PyThreadState_Init(tstate); PyEval_AcquireThread(tstate); tstate->interp->num_threads++; res = PyObject_Call(boot->func, boot->args, boot->keyw); diff --git a/Python/ceval.c b/Python/ceval.c index 71e6eb8ebcfd..f9ff4e09f17e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,8 +217,9 @@ _PyEval_FiniThreads(struct _ceval_runtime_state *ceval) } static inline void -exit_thread_if_finalizing(_PyRuntimeState *runtime, PyThreadState *tstate) +exit_thread_if_finalizing(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; /* _Py_Finalizing is protected by the GIL */ if (runtime->finalizing != NULL && !_Py_CURRENTLY_FINALIZING(runtime, tstate)) { drop_gil(&runtime->ceval, tstate); @@ -236,7 +237,7 @@ PyEval_AcquireLock(void) Py_FatalError("PyEval_AcquireLock: current thread state is NULL"); } take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); } void @@ -257,14 +258,15 @@ PyEval_AcquireThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_AcquireThread: NULL new thread state"); } + assert(tstate->interp != NULL); - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = tstate->interp->runtime; struct _ceval_runtime_state *ceval = &runtime->ceval; /* Check someone has called PyEval_InitThreads() to create the lock */ assert(gil_created(&ceval->gil)); take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("PyEval_AcquireThread: non-NULL old thread state"); } @@ -276,8 +278,9 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (tstate == NULL) { Py_FatalError("PyEval_ReleaseThread: NULL thread state"); } + assert(tstate->interp != NULL); - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = tstate->interp->runtime; PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); if (new_tstate != tstate) { Py_FatalError("PyEval_ReleaseThread: wrong thread state"); @@ -308,7 +311,7 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime) } /* Destroy all threads except the current one */ - _PyThreadState_DeleteExcept(runtime, current_tstate); + _PyThreadState_DeleteExcept(current_tstate); } /* This function is used to signal that async exceptions are waiting to be @@ -337,17 +340,18 @@ PyEval_SaveThread(void) void PyEval_RestoreThread(PyThreadState *tstate) { - _PyRuntimeState *runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - if (tstate == NULL) { Py_FatalError("PyEval_RestoreThread: NULL tstate"); } + assert(tstate->interp != NULL); + + _PyRuntimeState *runtime = tstate->interp->runtime; + struct _ceval_runtime_state *ceval = &runtime->ceval; assert(gil_created(&ceval->gil)); int err = errno; take_gil(ceval, tstate); - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); errno = err; _PyThreadState_Swap(&runtime->gilstate, tstate); @@ -1141,7 +1145,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) take_gil(ceval, tstate); /* Check if we should make a quick exit. */ - exit_thread_if_finalizing(runtime, tstate); + exit_thread_if_finalizing(tstate); if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("ceval: orphan tstate"); diff --git a/Python/import.c b/Python/import.c index ab7db6bc17f6..68d1f4003abc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -541,7 +541,8 @@ PyImport_Cleanup(void) _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings machinery. */ - _PyGC_DumpShutdownStats(&_PyRuntime); + _PyRuntimeState *runtime = interp->runtime; + _PyGC_DumpShutdownStats(runtime); /* Now, if there are any modules left alive, clear their globals to minimize potential leaks. All C extension modules actually end diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 10a28813faa8..6590ef8e9a27 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -545,7 +545,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, _PyEval_FiniThreads(&runtime->ceval); /* Auto-thread-state API */ - _PyGILState_Init(runtime, interp, tstate); + _PyGILState_Init(tstate); /* Create the GIL */ PyEval_InitThreads(); @@ -683,7 +683,7 @@ pyinit_config(_PyRuntimeState *runtime, } PyObject *sysmod; - status = _PySys_Create(runtime, interp, &sysmod); + status = _PySys_Create(interp, &sysmod); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -892,8 +892,9 @@ _Py_ReconfigureMainInterpreter(PyInterpreterState *interp) * non-zero return code. */ static PyStatus -pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp) +pyinit_main(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; if (!runtime->core_initialized) { return _PyStatus_ERR("runtime core not initialized"); } @@ -919,7 +920,7 @@ pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp) return _PyStatus_ERR("can't initialize time"); } - if (_PySys_InitMain(runtime, interp) < 0) { + if (_PySys_InitMain(interp) < 0) { return _PyStatus_ERR("can't finish initializing sys"); } @@ -999,7 +1000,7 @@ _Py_InitializeMain(void) _PyRuntimeState *runtime = &_PyRuntime; PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; - return pyinit_main(runtime, interp); + return pyinit_main(interp); } @@ -1026,7 +1027,7 @@ Py_InitializeFromConfig(const PyConfig *config) config = &interp->config; if (config->_init_main) { - status = pyinit_main(runtime, interp); + status = pyinit_main(interp); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1453,7 +1454,7 @@ new_interpreter(PyThreadState **tstate_p) } Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - if (_PySys_InitMain(runtime, interp) < 0) { + if (_PySys_InitMain(interp) < 0) { return _PyStatus_ERR("can't finish initializing sys"); } } diff --git a/Python/pystate.c b/Python/pystate.c index 833e0fb30dcb..2b7db0e48deb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -39,7 +39,6 @@ extern "C" { /* Forward declarations */ static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate); -static void _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate); static PyStatus @@ -192,6 +191,8 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) PyInterpreterState * PyInterpreterState_New(void) { + _PyRuntimeState *runtime = &_PyRuntime; + if (PySys_Audit("cpython.PyInterpreterState_New", NULL) < 0) { return NULL; } @@ -202,6 +203,9 @@ PyInterpreterState_New(void) } memset(interp, 0, sizeof(*interp)); + + interp->runtime = runtime; + interp->id_refcount = -1; interp->check_interval = 100; @@ -223,7 +227,6 @@ PyInterpreterState_New(void) #endif #endif - _PyRuntimeState *runtime = &_PyRuntime; struct pyinterpreters *interpreters = &runtime->interpreters; HEAD_LOCK(runtime); @@ -257,9 +260,11 @@ PyInterpreterState_New(void) } -static void -_PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) +void +PyInterpreterState_Clear(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; + if (PySys_Audit("cpython.PyInterpreterState_Clear", NULL) < 0) { PyErr_Clear(); } @@ -297,31 +302,25 @@ _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) // objects have been cleaned up at the point. } -void -PyInterpreterState_Clear(PyInterpreterState *interp) -{ - _PyInterpreterState_Clear(&_PyRuntime, interp); -} - static void -zapthreads(_PyRuntimeState *runtime, PyInterpreterState *interp) +zapthreads(PyInterpreterState *interp) { - PyThreadState *p; + PyThreadState *ts; /* No need to lock the mutex here because this should only happen when the threads are all really dead (XXX famous last words). */ - while ((p = interp->tstate_head) != NULL) { - _PyThreadState_Delete(runtime, p); + while ((ts = interp->tstate_head) != NULL) { + PyThreadState_Delete(ts); } } -static void -_PyInterpreterState_Delete(_PyRuntimeState *runtime, - PyInterpreterState *interp) +void +PyInterpreterState_Delete(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; struct pyinterpreters *interpreters = &runtime->interpreters; - zapthreads(runtime, interp); + zapthreads(interp); HEAD_LOCK(runtime); PyInterpreterState **p; for (p = &interpreters->head; ; p = &(*p)->next) { @@ -350,13 +349,6 @@ _PyInterpreterState_Delete(_PyRuntimeState *runtime, } -void -PyInterpreterState_Delete(PyInterpreterState *interp) -{ - _PyInterpreterState_Delete(&_PyRuntime, interp); -} - - /* * Delete all interpreter states except the main interpreter. If there * is a current interpreter state, it *must* be the main interpreter. @@ -383,8 +375,8 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) continue; } - _PyInterpreterState_Clear(runtime, interp); // XXX must activate? - zapthreads(runtime, interp); + PyInterpreterState_Clear(interp); // XXX must activate? + zapthreads(interp); if (interp->id_mutex != NULL) { PyThread_free_lock(interp->id_mutex); } @@ -497,7 +489,8 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) if (interp->id_mutex == NULL) { return; } - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; + _PyRuntimeState *runtime = interp->runtime; + struct _gilstate_runtime_state *gilstate = &runtime->gilstate; PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); assert(interp->id_refcount != 0); interp->id_refcount -= 1; @@ -559,7 +552,7 @@ threadstate_getframe(PyThreadState *self) static PyThreadState * new_threadstate(PyInterpreterState *interp, int init) { - _PyRuntimeState *runtime = &_PyRuntime; + _PyRuntimeState *runtime = interp->runtime; PyThreadState *tstate = (PyThreadState *)PyMem_RawMalloc(sizeof(PyThreadState)); if (tstate == NULL) { return NULL; @@ -615,7 +608,7 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->id = ++interp->tstate_next_unique_id; if (init) { - _PyThreadState_Init(runtime, tstate); + _PyThreadState_Init(tstate); } HEAD_LOCK(runtime); @@ -642,8 +635,9 @@ _PyThreadState_Prealloc(PyInterpreterState *interp) } void -_PyThreadState_Init(_PyRuntimeState *runtime, PyThreadState *tstate) +_PyThreadState_Init(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; _PyGILState_NoteThreadState(&runtime->gilstate, tstate); } @@ -808,7 +802,7 @@ PyThreadState_Clear(PyThreadState *tstate) /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate) { if (tstate == NULL) { Py_FatalError("PyThreadState_Delete: NULL tstate"); @@ -817,6 +811,7 @@ tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) if (interp == NULL) { Py_FatalError("PyThreadState_Delete: NULL interp"); } + _PyRuntimeState *runtime = interp->runtime; HEAD_LOCK(runtime); if (tstate->prev) tstate->prev->next = tstate->next; @@ -832,9 +827,10 @@ tstate_delete_common(_PyRuntimeState *runtime, PyThreadState *tstate) } -static void -_PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate) +void +PyThreadState_Delete(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; struct _gilstate_runtime_state *gilstate = &runtime->gilstate; if (tstate == _PyRuntimeGILState_GetThreadState(gilstate)) { Py_FatalError("PyThreadState_Delete: tstate is still current"); @@ -844,14 +840,7 @@ _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstate) { PyThread_tss_set(&gilstate->autoTSSkey, NULL); } - tstate_delete_common(runtime, tstate); -} - - -void -PyThreadState_Delete(PyThreadState *tstate) -{ - _PyThreadState_Delete(&_PyRuntime, tstate); + tstate_delete_common(tstate); } @@ -863,7 +852,7 @@ _PyThreadState_DeleteCurrent(_PyRuntimeState *runtime) if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - tstate_delete_common(runtime, tstate); + tstate_delete_common(tstate); if (gilstate->autoInterpreterState && PyThread_tss_get(&gilstate->autoTSSkey) == tstate) { @@ -888,9 +877,10 @@ PyThreadState_DeleteCurrent() * be kept in those other interpreteres. */ void -_PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) +_PyThreadState_DeleteExcept(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; + _PyRuntimeState *runtime = interp->runtime; PyThreadState *p, *next, *garbage; HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of @@ -1129,8 +1119,9 @@ _PyThread_CurrentFrames(void) static int PyThreadState_IsCurrent(PyThreadState *tstate) { + _PyRuntimeState *runtime = tstate->interp->runtime; /* Must be the tstate for this thread */ - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; + struct _gilstate_runtime_state *gilstate = &runtime->gilstate; assert(_PyGILState_GetThisThreadState(gilstate) == tstate); return tstate == _PyRuntimeGILState_GetThreadState(gilstate); } @@ -1139,12 +1130,14 @@ PyThreadState_IsCurrent(PyThreadState *tstate) Py_Initialize/Py_FinalizeEx */ void -_PyGILState_Init(_PyRuntimeState *runtime, - PyInterpreterState *interp, PyThreadState *tstate) +_PyGILState_Init(PyThreadState *tstate) { /* must init with valid states */ - assert(interp != NULL); assert(tstate != NULL); + PyInterpreterState *interp = tstate->interp; + assert(interp != NULL); + _PyRuntimeState *runtime = interp->runtime; + assert(runtime != NULL); struct _gilstate_runtime_state *gilstate = &runtime->gilstate; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 12b1bd7711d5..97bff94d8b41 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -120,8 +120,9 @@ should_audit(void) if (!ts) { return 0; } - PyInterpreterState *is = ts ? ts->interp : NULL; - return _PyRuntime.audit_hook_head + PyInterpreterState *is = ts->interp; + _PyRuntimeState *runtime = is->runtime; + return runtime->audit_hook_head || (is && is->audit_hooks) || PyDTrace_AUDIT_ENABLED(); } @@ -280,8 +281,8 @@ void _PySys_ClearAuditHooks(void) { PySys_Audit("cpython._PySys_ClearAuditHooks", NULL); PyErr_Clear(); - _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head, *n; - _PyRuntime.audit_hook_head = NULL; + _Py_AuditHookEntry *e = runtime->audit_hook_head, *n; + runtime->audit_hook_head = NULL; while (e) { n = e->next; PyMem_RawFree(e); @@ -292,6 +293,7 @@ void _PySys_ClearAuditHooks(void) { int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) { + _PyRuntimeState *runtime = &_PyRuntime; /* Invoke existing audit hooks to allow them an opportunity to abort. */ /* Cannot invoke hooks until we are initialized */ if (Py_IsInitialized()) { @@ -305,10 +307,10 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) } } - _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head; + _Py_AuditHookEntry *e = runtime->audit_hook_head; if (!e) { e = (_Py_AuditHookEntry*)PyMem_RawMalloc(sizeof(_Py_AuditHookEntry)); - _PyRuntime.audit_hook_head = e; + runtime->audit_hook_head = e; } else { while (e->next) e = e->next; @@ -2413,8 +2415,9 @@ static PyStructSequence_Desc flags_desc = { }; static PyObject* -make_flags(_PyRuntimeState *runtime, PyInterpreterState *interp) +make_flags(PyInterpreterState *interp) { + _PyRuntimeState *runtime = interp->runtime; int pos = 0; PyObject *seq; const PyPreConfig *preconfig = &runtime->preconfig; @@ -2633,8 +2636,7 @@ static struct PyModuleDef sysmodule = { } while (0) static PyStatus -_PySys_InitCore(_PyRuntimeState *runtime, PyInterpreterState *interp, - PyObject *sysdict) +_PySys_InitCore(PyInterpreterState *interp, PyObject *sysdict) { PyObject *version_info; int res; @@ -2728,7 +2730,7 @@ _PySys_InitCore(_PyRuntimeState *runtime, PyInterpreterState *interp, } } /* Set flags to their default values (updated by _PySys_InitMain()) */ - SET_SYS_FROM_STRING("flags", make_flags(runtime, interp)); + SET_SYS_FROM_STRING("flags", make_flags(interp)); #if defined(MS_WINDOWS) /* getwindowsversion */ @@ -2849,7 +2851,7 @@ sys_create_xoptions_dict(const PyConfig *config) int -_PySys_InitMain(_PyRuntimeState *runtime, PyInterpreterState *interp) +_PySys_InitMain(PyInterpreterState *interp) { PyObject *sysdict = interp->sysdict; const PyConfig *config = &interp->config; @@ -2903,7 +2905,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyInterpreterState *interp) #undef SET_SYS_FROM_WSTR /* Set flags to their final values */ - SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags(runtime, interp)); + SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags(interp)); /* prevent user from creating new instances */ FlagsType.tp_init = NULL; FlagsType.tp_new = NULL; @@ -2970,8 +2972,7 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict) /* Create sys module without all attributes: _PySys_InitMain() should be called later to add remaining attributes. */ PyStatus -_PySys_Create(_PyRuntimeState *runtime, PyInterpreterState *interp, - PyObject **sysmod_p) +_PySys_Create(PyInterpreterState *interp, PyObject **sysmod_p) { PyObject *modules = PyDict_New(); if (modules == NULL) { @@ -3000,7 +3001,7 @@ _PySys_Create(_PyRuntimeState *runtime, PyInterpreterState *interp, return status; } - status = _PySys_InitCore(runtime, interp, sysdict); + status = _PySys_InitCore(interp, sysdict); if (_PyStatus_EXCEPTION(status)) { return status; }

1 0

bpo-37029: keep usable_arenas in sorted order without searching (#13612)
by Tim Peters 31 May '19

31 May '19

https://github.com/python/cpython/commit/1c263e39c4ed28225a7dc8ca1f24953225… commit: 1c263e39c4ed28225a7dc8ca1f24953225ac48ca branch: master author: Tim Peters <tim.peters(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-05-31T21:16:04-05:00 summary: bpo-37029: keep usable_arenas in sorted order without searching (#13612) This adds a vector of "search fingers" so that usable_arenas can be kept in sorted order (by number of free pools) via constant-time operations instead of linear search. This should reduce worst-case time for reclaiming a great many objects from O(A**2) to O(A), where A is the number of arenas. See bpo-37029. files: A Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst M Objects/obmalloc.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst new file mode 100644 index 000000000000..c18f5d23eaa2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-28-17-02-46.bpo-37029.MxpgfJ.rst @@ -0,0 +1 @@ +Freeing a great many small objects could take time quadratic in the number of arenas, due to using linear search to keep ``obmalloc.c``'s list of usable arenas sorted by order of number of free memory pools. This is accomplished without search now, leaving the worst-case time linear in the number of arenas. For programs where this quite visibly matters (typically with more than 100 thousand small objects alive simultaneously), this can greatly reduce the time needed to release their memory. \ No newline at end of file diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f54856dcfe71..fc7bef619946 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -918,6 +918,11 @@ static int running_on_valgrind = -1; #define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ #define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK +#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE) +#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE +# error "arena size not an exact multiple of pool size" +#endif + /* * -- End of tunable settings section -- */ @@ -1155,6 +1160,18 @@ usable_arenas Note that an arena_object associated with an arena all of whose pools are currently in use isn't on either list. + +Changed in Python 3.8: keeping usable_arenas sorted by number of free pools +used to be done by one-at-a-time linear search when an arena's number of +free pools changed. That could, overall, consume time quadratic in the +number of arenas. That didn't really matter when there were only a few +hundred arenas (typical!), but could be a timing disaster when there were +hundreds of thousands. See bpo-37029. + +Now we have a vector of "search fingers" to eliminate the need to search: +nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas +with nfp free pools. This is NULL if and only if there is no arena with +nfp free pools in usable_arenas. */ /* Array of objects used to track chunks of memory (arenas). */ @@ -1172,6 +1189,9 @@ static struct arena_object* unused_arena_objects = NULL; */ static struct arena_object* usable_arenas = NULL; +/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */ +static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; + /* How many arena_objects do we initially allocate? * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the * `arenas` vector. @@ -1281,8 +1301,7 @@ new_arena(void) /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ arenaobj->pool_address = (block*)arenaobj->address; - arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; - assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); + arenaobj->nfreepools = MAX_POOLS_IN_ARENA; excess = (uint)(arenaobj->address & POOL_SIZE_MASK); if (excess != 0) { --arenaobj->nfreepools; @@ -1478,22 +1497,32 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) } usable_arenas->nextarena = usable_arenas->prevarena = NULL; + assert(nfp2lasta[usable_arenas->nfreepools] == NULL); + nfp2lasta[usable_arenas->nfreepools] = usable_arenas; } assert(usable_arenas->address != 0); + /* This arena already had the smallest nfreepools value, so decreasing + * nfreepools doesn't change that, and we don't need to rearrange the + * usable_arenas list. However, if the arena becomes wholly allocated, + * we need to remove its arena_object from usable_arenas. + */ + assert(usable_arenas->nfreepools > 0); + if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) { + /* It's the last of this size, so there won't be any. */ + nfp2lasta[usable_arenas->nfreepools] = NULL; + } + /* If any free pools will remain, it will be the new smallest. */ + if (usable_arenas->nfreepools > 1) { + assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL); + nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas; + } + /* Try to get a cached free pool. */ pool = usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ usable_arenas->freepools = pool->nextpool; - - /* This arena already had the smallest nfreepools - * value, so decreasing nfreepools doesn't change - * that, and we don't need to rearrange the - * usable_arenas list. However, if the arena has - * become wholly allocated, we need to remove its - * arena_object from usable_arenas. - */ --usable_arenas->nfreepools; if (usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ @@ -1501,7 +1530,6 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) assert(usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas); - usable_arenas = usable_arenas->nextarena; if (usable_arenas != NULL) { usable_arenas->prevarena = NULL; @@ -1709,7 +1737,23 @@ pymalloc_free(void *ctx, void *p) ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; - nf = ++ao->nfreepools; + nf = ao->nfreepools; + /* If this is the rightmost arena with this number of free pools, + * nfp2lasta[nf] needs to change. Caution: if nf is 0, there + * are no arenas in usable_arenas with that value. + */ + struct arena_object* lastnf = nfp2lasta[nf]; + assert((nf == 0 && lastnf == NULL) || + (nf > 0 && + lastnf != NULL && + lastnf->nfreepools == nf && + (lastnf->nextarena == NULL || + nf < lastnf->nextarena->nfreepools))); + if (lastnf == ao) { /* it is the rightmost */ + struct arena_object* p = ao->prevarena; + nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL; + } + ao->nfreepools = ++nf; /* All the rest is arena management. We just freed * a pool, and there are 4 cases for arena mgmt: @@ -1777,6 +1821,9 @@ pymalloc_free(void *ctx, void *p) usable_arenas->prevarena = ao; usable_arenas = ao; assert(usable_arenas->address != 0); + if (nfp2lasta[1] == NULL) { + nfp2lasta[1] = ao; + } goto success; } @@ -1788,14 +1835,23 @@ pymalloc_free(void *ctx, void *p) * a few un-scientific tests, it seems like this * approach allowed a lot more memory to be freed. */ - if (ao->nextarena == NULL || - nf <= ao->nextarena->nfreepools) { + /* If this is the only arena with nf, record that. */ + if (nfp2lasta[nf] == NULL) { + nfp2lasta[nf] = ao; + } /* else the rightmost with nf doesn't change */ + /* If this was the rightmost of the old size, it remains in place. */ + if (ao == lastnf) { /* Case 4. Nothing to do. */ goto success; } - /* Case 3: We have to move the arena towards the end - * of the list, because it has more free pools than - * the arena to its right. + /* If ao were the only arena in the list, the last block would have + * gotten us out. + */ + assert(ao->nextarena != NULL); + + /* Case 3: We have to move the arena towards the end of the list, + * because it has more free pools than the arena to its right. It needs + * to move to follow lastnf. * First unlink ao from usable_arenas. */ if (ao->prevarena != NULL) { @@ -1809,24 +1865,13 @@ pymalloc_free(void *ctx, void *p) usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; - - /* Locate the new insertion point by iterating over - * the list, using our nextarena pointer. - */ - while (ao->nextarena != NULL && nf > ao->nextarena->nfreepools) { - ao->prevarena = ao->nextarena; - ao->nextarena = ao->nextarena->nextarena; - } - - /* Insert ao at this point. */ - assert(ao->nextarena == NULL || ao->prevarena == ao->nextarena->prevarena); - assert(ao->prevarena->nextarena == ao->nextarena); - - ao->prevarena->nextarena = ao; + /* And insert after lastnf. */ + ao->prevarena = lastnf; + ao->nextarena = lastnf->nextarena; if (ao->nextarena != NULL) { ao->nextarena->prevarena = ao; } - + lastnf->nextarena = ao; /* Verify that the swaps worked. */ assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools);

1 0

bpo-12202: Properly check MsiSummaryInfoGetProperty() calls in msilib (GH-13711)
by Berker Peksag 31 May '19

31 May '19

https://github.com/python/cpython/commit/549e55a3086d04c13da9b6f33214f63996… commit: 549e55a3086d04c13da9b6f33214f6399681292a branch: master author: Zackery Spytz <zspytz(a)gmail.com> committer: Berker Peksag <berker.peksag(a)gmail.com> date: 2019-06-01T03:16:20+03:00 summary: bpo-12202: Properly check MsiSummaryInfoGetProperty() calls in msilib (GH-13711) files: A Misc/NEWS.d/next/Library/2019-05-31-15-53-34.bpo-12202.nobzc9.rst M Lib/test/test_msilib.py M PC/_msi.c diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index 265eaea59b5f..fa0be581613d 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -85,6 +85,7 @@ def test_get_property_vt_empty(self): def test_directory_start_component_keyfile(self): db, db_path = init_database() + self.addCleanup(unlink, db_path) self.addCleanup(db.Close) feature = msilib.Feature(db, 0, 'Feature', 'A feature', 'Python') cab = msilib.CAB('CAB') @@ -92,6 +93,14 @@ def test_directory_start_component_keyfile(self): 'SourceDir', 0) dir.start_component(None, feature, None, 'keyfile') + def test_getproperty_uninitialized_var(self): + db, db_path = init_database() + self.addCleanup(unlink, db_path) + self.addCleanup(db.Close) + si = db.GetSummaryInformation(0) + with self.assertRaises(msilib.MSIError): + si.GetProperty(-1) + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Misc/NEWS.d/next/Library/2019-05-31-15-53-34.bpo-12202.nobzc9.rst b/Misc/NEWS.d/next/Library/2019-05-31-15-53-34.bpo-12202.nobzc9.rst new file mode 100644 index 000000000000..1e561970445f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-31-15-53-34.bpo-12202.nobzc9.rst @@ -0,0 +1,2 @@ +Fix the error handling in :meth:`msilib.SummaryInformation.GetProperty`. Patch +by Zackery Spytz. diff --git a/PC/_msi.c b/PC/_msi.c index 4c8df5b42b95..accbe7a72069 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -571,6 +571,9 @@ summary_getproperty(msiobj* si, PyObject *args) status = MsiSummaryInfoGetProperty(si->h, field, &type, &ival, &fval, sval, &ssize); } + if (status != ERROR_SUCCESS) { + return msierror(status); + } switch(type) { case VT_I2:

1 0