Mailman 3 September 2023 - Python-checkins

Make test_fcntl quiet (#108758)
by vstinner Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/23f54c120067c96973619b9501fe4dff1b… commit: 23f54c120067c96973619b9501fe4dff1b055188 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-01T14:59:09+02:00 summary: Make test_fcntl quiet (#108758) Running test_fcntl logs two "struct.pack: ..." lines because multiprocessing imports test_fcntl twice with test.support.verbose=1. Move get_lockdata() inside TestFcntl test case and only call it where it's needed, to stop logging these lines. files: M Lib/test/test_fcntl.py diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 5da75615b41d7..203dd6fe57dcd 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -16,37 +16,6 @@ -def get_lockdata(): - try: - os.O_LARGEFILE - except AttributeError: - start_len = "ll" - else: - start_len = "qq" - - if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) - or sys.platform == 'darwin'): - if struct.calcsize('l') == 8: - off_t = 'l' - pid_t = 'i' - else: - off_t = 'lxxxx' - pid_t = 'l' - lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0, - fcntl.F_WRLCK, 0) - elif sys.platform.startswith('gnukfreebsd'): - lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0) - elif sys.platform in ['hp-uxB', 'unixware7']: - lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) - else: - lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) - if lockdata: - if verbose: - print('struct.pack: ', repr(lockdata)) - return lockdata - -lockdata = get_lockdata() - class BadFile: def __init__(self, fn): self.fn = fn @@ -78,12 +47,43 @@ def tearDown(self): self.f.close() unlink(TESTFN) + @staticmethod + def get_lockdata(): + try: + os.O_LARGEFILE + except AttributeError: + start_len = "ll" + else: + start_len = "qq" + + if (sys.platform.startswith(('netbsd', 'freebsd', 'openbsd')) + or sys.platform == 'darwin'): + if struct.calcsize('l') == 8: + off_t = 'l' + pid_t = 'i' + else: + off_t = 'lxxxx' + pid_t = 'l' + lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0, + fcntl.F_WRLCK, 0) + elif sys.platform.startswith('gnukfreebsd'): + lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0) + elif sys.platform in ['hp-uxB', 'unixware7']: + lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) + else: + lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) + if lockdata: + if verbose: + print('struct.pack: ', repr(lockdata)) + return lockdata + def test_fcntl_fileno(self): # the example from the library docs self.f = open(TESTFN, 'wb') rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) if verbose: print('Status from fcntl with O_NONBLOCK: ', rv) + lockdata = self.get_lockdata() rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata) if verbose: print('String from fcntl with F_SETLKW: ', repr(rv)) @@ -95,6 +95,7 @@ def test_fcntl_file_descriptor(self): rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK) if verbose: print('Status from fcntl with O_NONBLOCK: ', rv) + lockdata = self.get_lockdata() rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata) if verbose: print('String from fcntl with F_SETLKW: ', repr(rv))

1 0

[3.11] gh-101100: Fix sphinx warnings in `tutorial/appendix.rst` (GH-108750) (#108757)
by hugovk Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/d3476dcb8488b4d3d0d4343c03bbadd9a3… commit: d3476dcb8488b4d3d0d4343c03bbadd9a329392d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-01T15:09:43+03:00 summary: [3.11] gh-101100: Fix sphinx warnings in `tutorial/appendix.rst` (GH-108750) (#108757) Co-authored-by: Nikita Sobolev <mail(a)sobolevn.me> Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/tools/.nitignore M Doc/tutorial/appendix.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ab7f4a8dc618b..6dae72a9dfeff 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -157,7 +157,6 @@ Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst -Doc/tutorial/appendix.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 241a812037469..588591fcdb726 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -101,8 +101,8 @@ in the script:: The Customization Modules ------------------------- -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location +Python provides two hooks to let you customize it: :index:`sitecustomize` and +:index:`usercustomize`. To see how it works, you need first to find the location of your user site-packages directory. Start Python and run this code:: >>> import site @@ -113,9 +113,9 @@ Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless it is started with the :option:`-s` option to disable the automatic import. -:mod:`sitecustomize` works in the same way, but is typically created by an +:index:`sitecustomize` works in the same way, but is typically created by an administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` +imported before :index:`usercustomize`. See the documentation of the :mod:`site` module for more details.

1 0

gh-101100: Fix sphinx warnings in `tutorial/appendix.rst` (#108750)
by hugovk Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/3047f09490ae63f25d57efe1d14a9a65d9… commit: 3047f09490ae63f25d57efe1d14a9a65d9b5f6db branch: main author: Nikita Sobolev <mail(a)sobolevn.me> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-01T10:57:34Z summary: gh-101100: Fix sphinx warnings in `tutorial/appendix.rst` (#108750) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/tools/.nitignore M Doc/tutorial/appendix.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 67641f73e4649..1d89c9c683a07 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -155,7 +155,6 @@ Doc/reference/datamodel.rst Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst -Doc/tutorial/appendix.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 241a812037469..588591fcdb726 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -101,8 +101,8 @@ in the script:: The Customization Modules ------------------------- -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location +Python provides two hooks to let you customize it: :index:`sitecustomize` and +:index:`usercustomize`. To see how it works, you need first to find the location of your user site-packages directory. Start Python and run this code:: >>> import site @@ -113,9 +113,9 @@ Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless it is started with the :option:`-s` option to disable the automatic import. -:mod:`sitecustomize` works in the same way, but is typically created by an +:index:`sitecustomize` works in the same way, but is typically created by an administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` +imported before :index:`usercustomize`. See the documentation of the :mod:`site` module for more details.

1 0

[3.11] gh-101100: Fix sphinx warnings in `tutorial/classes.rst` (GH-108746) (#108755)
by hugovk Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/3b143468995ca9e6e1ecfa69b5dc8fd587… commit: 3b143468995ca9e6e1ecfa69b5dc8fd587fcc33a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-01T10:57:30Z summary: [3.11] gh-101100: Fix sphinx warnings in `tutorial/classes.rst` (GH-108746) (#108755) Co-authored-by: Nikita Sobolev <mail(a)sobolevn.me> files: M Doc/tools/.nitignore M Doc/tutorial/classes.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 871a48282523c..ab7f4a8dc618b 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -158,7 +158,6 @@ Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst Doc/tutorial/appendix.rst -Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index e4002cfa35193..46dfb378590ff 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,7 +276,7 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`__doc__` is also a valid attribute, returning the docstring belonging to +:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class

1 0

gh-101100: Fix sphinx warnings in `tutorial/classes.rst` (#108746)
by hugovk Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/e775601ef1ada83a80e1539d0bcd5509de… commit: e775601ef1ada83a80e1539d0bcd5509deadee14 branch: main author: Nikita Sobolev <mail(a)sobolevn.me> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-01T13:44:43+03:00 summary: gh-101100: Fix sphinx warnings in `tutorial/classes.rst` (#108746) files: M Doc/tools/.nitignore M Doc/tutorial/classes.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 22df73ef0cfb0..67641f73e4649 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -156,7 +156,6 @@ Doc/reference/expressions.rst Doc/reference/import.rst Doc/reference/simple_stmts.rst Doc/tutorial/appendix.rst -Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst Doc/tutorial/introduction.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 91a3b73d2b55a..7b92e1a51b6e6 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,7 +276,7 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`__doc__` is also a valid attribute, returning the docstring belonging to +:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class

1 0

gh-108634: PyInterpreterState_New() no longer calls Py_FatalError() (#108748)
by vstinner Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/b936cf4fe084474970ea43dba24386f1b0… commit: b936cf4fe084474970ea43dba24386f1b08e75ce branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-01T12:43:30+02:00 summary: gh-108634: PyInterpreterState_New() no longer calls Py_FatalError() (#108748) pycore_create_interpreter() now returns a status, rather than calling Py_FatalError(). * PyInterpreterState_New() now calls Py_ExitStatusException() instead of calling Py_FatalError() directly. * Replace Py_FatalError() with PyStatus in init_interpreter() and _PyObject_InitState(). * _PyErr_SetFromPyStatus() now raises RuntimeError, instead of ValueError. It can now call PyErr_NoMemory(), raise MemoryError, if it detects _PyStatus_NO_MEMORY() error message. files: M Include/internal/pycore_initconfig.h M Include/internal/pycore_interp.h M Include/internal/pycore_object.h M Objects/object.c M Python/initconfig.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 6439101ff390a..c86988234f6a0 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -22,7 +22,7 @@ struct pyruntimestate; #endif #define _PyStatus_OK() \ - (PyStatus){._type = _PyStatus_TYPE_OK,} + (PyStatus){._type = _PyStatus_TYPE_OK} /* other fields are set to 0 */ #define _PyStatus_ERR(ERR_MSG) \ (PyStatus){ \ @@ -30,7 +30,8 @@ struct pyruntimestate; .func = _PyStatus_GET_FUNC(), \ .err_msg = (ERR_MSG)} /* other fields are set to 0 */ -#define _PyStatus_NO_MEMORY() _PyStatus_ERR("memory allocation failed") +#define _PyStatus_NO_MEMORY_ERRMSG "memory allocation failed" +#define _PyStatus_NO_MEMORY() _PyStatus_ERR(_PyStatus_NO_MEMORY_ERRMSG) #define _PyStatus_EXIT(EXITCODE) \ (PyStatus){ \ ._type = _PyStatus_TYPE_EXIT, \ @@ -45,7 +46,7 @@ struct pyruntimestate; do { (err).func = _PyStatus_GET_FUNC(); } while (0) // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); +PyAPI_FUNC(void) _PyErr_SetFromPyStatus(PyStatus status); /* --- PyWideStringList ------------------------------------------------ */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 9a75b002e5bc1..f171c546efd53 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -311,6 +311,10 @@ might not be allowed in the current interpreter (i.e. os.fork() would fail). extern int _PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature); +PyAPI_FUNC(PyStatus) _PyInterpreterState_New( + PyThreadState *tstate, + PyInterpreterState **pinterp); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index d842816e67355..daa06ebfbf91a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -182,7 +182,7 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); -extern void _PyObject_InitState(PyInterpreterState *interp); +extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); diff --git a/Objects/object.c b/Objects/object.c index a4d7111a686bd..7aeda50e9b275 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2034,7 +2034,7 @@ PyObject _Py_NotImplementedStruct = { }; -void +PyStatus _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS @@ -2048,9 +2048,10 @@ _PyObject_InitState(PyInterpreterState *interp) _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, NULL, NULL, &alloc); if (REFCHAIN(interp) == NULL) { - Py_FatalError("_PyObject_InitState() memory allocation failure"); + return _PyStatus_NO_MEMORY(); } #endif + return _PyStatus_OK(); } void diff --git a/Python/initconfig.c b/Python/initconfig.c index 3281b3c6aea45..f9c5c64f9c875 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -335,21 +335,34 @@ int PyStatus_IsExit(PyStatus status) int PyStatus_Exception(PyStatus status) { return _PyStatus_EXCEPTION(status); } -PyObject* +void _PyErr_SetFromPyStatus(PyStatus status) { if (!_PyStatus_IS_ERROR(status)) { PyErr_Format(PyExc_SystemError, - "%s() expects an error PyStatus", - _PyStatus_GET_FUNC()); + "_PyErr_SetFromPyStatus() status is not an error"); + return; } - else if (status.func) { - PyErr_Format(PyExc_ValueError, "%s: %s", status.func, status.err_msg); + + const char *err_msg = status.err_msg; + if (err_msg == NULL || strlen(err_msg) == 0) { + PyErr_Format(PyExc_SystemError, + "_PyErr_SetFromPyStatus() status has no error message"); + return; + } + + if (strcmp(err_msg, _PyStatus_NO_MEMORY_ERRMSG) == 0) { + PyErr_NoMemory(); + return; + } + + const char *func = status.func; + if (func) { + PyErr_Format(PyExc_RuntimeError, "%s: %s", func, err_msg); } else { - PyErr_Format(PyExc_ValueError, "%s", status.err_msg); + PyErr_Format(PyExc_RuntimeError, "%s", err_msg); } - return NULL; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ee5d4981da51c..64c74f433f222 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -629,10 +629,12 @@ pycore_create_interpreter(_PyRuntimeState *runtime, PyThreadState **tstate_p) { PyStatus status; - PyInterpreterState *interp = PyInterpreterState_New(); - if (interp == NULL) { - return _PyStatus_ERR("can't make main interpreter"); + PyInterpreterState *interp; + status = _PyInterpreterState_New(NULL, &interp); + if (_PyStatus_EXCEPTION(status)) { + return status; } + assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); status = _PyConfig_Copy(&interp->config, src_config); diff --git a/Python/pystate.c b/Python/pystate.c index 4a8808f700eaa..46ea2c4f678db 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -426,13 +426,11 @@ init_runtime(_PyRuntimeState *runtime, Py_ssize_t unicode_next_index, PyThread_type_lock locks[NUMLOCKS]) { - if (runtime->_initialized) { - Py_FatalError("runtime already initialized"); - } - assert(!runtime->preinitializing && - !runtime->preinitialized && - !runtime->core_initialized && - !runtime->initialized); + assert(!runtime->preinitializing); + assert(!runtime->preinitialized); + assert(!runtime->core_initialized); + assert(!runtime->initialized); + assert(!runtime->_initialized); runtime->open_code_hook = open_code_hook; runtime->open_code_userdata = open_code_userdata; @@ -476,6 +474,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); + assert(!runtime->_initialized); } if (gilstate_tss_init(runtime) != 0) { @@ -647,14 +646,14 @@ free_interpreter(PyInterpreterState *interp) main interpreter. We fix those fields here, in addition to the other dynamically initialized fields. */ -static void +static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, PyInterpreterState *next, PyThread_type_lock pending_lock) { if (interp->_initialized) { - Py_FatalError("interpreter already initialized"); + return _PyStatus_ERR("interpreter already initialized"); } assert(runtime != NULL); @@ -675,7 +674,10 @@ init_interpreter(PyInterpreterState *interp, memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } - _PyObject_InitState(interp); + PyStatus status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); @@ -701,42 +703,44 @@ init_interpreter(PyInterpreterState *interp, } interp->f_opcode_trace_set = false; interp->_initialized = 1; + return _PyStatus_OK(); } -PyInterpreterState * -PyInterpreterState_New(void) + +PyStatus +_PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) { - PyInterpreterState *interp; + *pinterp = NULL; + + // Don't get runtime from tstate since tstate can be NULL _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); - /* tstate is NULL when Py_InitializeFromConfig() calls - PyInterpreterState_New() to create the main interpreter. */ - if (_PySys_Audit(tstate, "cpython.PyInterpreterState_New", NULL) < 0) { - return NULL; + // tstate is NULL when pycore_create_interpreter() calls + // _PyInterpreterState_New() to create the main interpreter. + if (tstate != NULL) { + if (_PySys_Audit(tstate, "cpython.PyInterpreterState_New", NULL) < 0) { + return _PyStatus_ERR("sys.audit failed"); + } } PyThread_type_lock pending_lock = PyThread_allocate_lock(); if (pending_lock == NULL) { - if (tstate != NULL) { - _PyErr_NoMemory(tstate); - } - return NULL; + return _PyStatus_NO_MEMORY(); } - /* Don't get runtime from tstate since tstate can be NULL. */ - struct pyinterpreters *interpreters = &runtime->interpreters; - /* We completely serialize creation of multiple interpreters, since it simplifies things here and blocking concurrent calls isn't a problem. Regardless, we must fully block subinterpreter creation until after the main interpreter is created. */ HEAD_LOCK(runtime); + struct pyinterpreters *interpreters = &runtime->interpreters; int64_t id = interpreters->next_id; interpreters->next_id += 1; // Allocate the interpreter and add it to the runtime state. + PyInterpreterState *interp; + PyStatus status; PyInterpreterState *old_head = interpreters->head; if (old_head == NULL) { // We are creating the main interpreter. @@ -755,36 +759,59 @@ PyInterpreterState_New(void) interp = alloc_interpreter(); if (interp == NULL) { + status = _PyStatus_NO_MEMORY(); goto error; } // Set to _PyInterpreterState_INIT. - memcpy(interp, &initial._main_interpreter, - sizeof(*interp)); + memcpy(interp, &initial._main_interpreter, sizeof(*interp)); if (id < 0) { /* overflow or Py_Initialize() not called yet! */ - if (tstate != NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "failed to get an interpreter ID"); - } + status = _PyStatus_ERR("failed to get an interpreter ID"); goto error; } } interpreters->head = interp; - init_interpreter(interp, runtime, id, old_head, pending_lock); + status = init_interpreter(interp, runtime, + id, old_head, pending_lock); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } + pending_lock = NULL; HEAD_UNLOCK(runtime); - return interp; + + assert(interp != NULL); + *pinterp = interp; + return _PyStatus_OK(); error: HEAD_UNLOCK(runtime); - PyThread_free_lock(pending_lock); + if (pending_lock != NULL) { + PyThread_free_lock(pending_lock); + } if (interp != NULL) { free_interpreter(interp); } - return NULL; + return status; +} + + +PyInterpreterState * +PyInterpreterState_New(void) +{ + // tstate can be NULL + PyThreadState *tstate = current_fast_get(&_PyRuntime); + + PyInterpreterState *interp; + PyStatus status = _PyInterpreterState_New(tstate, &interp); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + assert(interp != NULL); + return interp; }

1 0

gh-108727: Fix segfault due to missing tp_dealloc definition for CounterOptimizer_Type (GH-108734)
by markshannon Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/844f4c2e12a7c637d1de93dbbb0718be06… commit: 844f4c2e12a7c637d1de93dbbb0718be06553510 branch: main author: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> committer: markshannon <mark(a)hotpy.org> date: 2023-09-01T10:16:09+01:00 summary: gh-108727: Fix segfault due to missing tp_dealloc definition for CounterOptimizer_Type (GH-108734) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst M Lib/test/test_capi/test_misc.py M Python/optimizer.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4148f15b2aa66..004ce39769655 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2284,6 +2284,13 @@ def clear_executors(func): class TestOptimizerAPI(unittest.TestCase): + def test_get_counter_optimizer_dealloc(self): + # See gh-108727 + def f(): + _testinternalcapi.get_counter_optimizer() + + f() + def test_get_set_optimizer(self): old = _testinternalcapi.get_optimizer() opt = _testinternalcapi.get_counter_optimizer() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst new file mode 100644 index 0000000000000..34959ae3bb99a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-31-21-29-28.gh-issue-108727.blNRGM.rst @@ -0,0 +1,2 @@ +Define ``tp_dealloc`` for ``CounterOptimizer_Type``. This fixes a segfault +on deallocation. diff --git a/Python/optimizer.c b/Python/optimizer.c index c311a03fa59a2..7472f52c50b76 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -289,6 +289,7 @@ static PyTypeObject CounterOptimizer_Type = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_methods = counter_methods, + .tp_dealloc = (destructor)PyObject_Del, }; PyObject *

1 0

[3.11] gh-104372: Drop the GIL around the vfork() call. (#104782) (#104958)
by gpshead Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/6ba1234c1c643743be15090ecf8da39137… commit: 6ba1234c1c643743be15090ecf8da39137c8cbac branch: 3.11 author: Gregory P. Smith <greg(a)krypto.org> committer: gpshead <greg(a)krypto.org> date: 2023-09-01T08:53:06Z summary: [3.11] gh-104372: Drop the GIL around the vfork() call. (#104782) (#104958) gh-104372: Drop the GIL around the vfork() call. (#104782) On Linux where the `subprocess` module can use the `vfork` syscall for faster spawning, prevent the parent process from blocking other threads by dropping the GIL while it waits for the vfork'ed child process `exec` outcome. This prevents spawning a binary from a slow filesystem from blocking the rest of the application. Fixes #104372. (cherry picked from commit d08679212d9af52dd074cd4a6abb440edb944c9c) files: A Misc/NEWS.d/next/Library/2023-05-22-18-39-53.gh-issue-104372.7tDRaK.rst M Doc/library/subprocess.rst M Modules/_posixsubprocess.c diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 6ef3c34b4fb81..377f6938701da 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -57,10 +57,13 @@ underlying :class:`Popen` interface can be used directly. and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` instead of *capture_output*. - The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout - expires, the child process will be killed and waited for. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. + A *timeout* may be specified in seconds, it is internally passed on to + :meth:`Popen.communicate`. If the timeout expires, the child process will be + killed and waited for. The :exc:`TimeoutExpired` exception will be + re-raised after the child process has terminated. The initial process + creation itself cannot be interrupted on many platform APIs so you are not + guaranteed to see a timeout exception until at least after however long + process creation takes. The *input* argument is passed to :meth:`Popen.communicate` and thus to the subprocess's stdin. If used it must be a byte sequence, or a string if @@ -736,7 +739,7 @@ arguments. code. All of the functions and methods that accept a *timeout* parameter, such as -:func:`call` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if +:func:`run` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if the timeout expires before the process exits. Exceptions defined in this module all inherit from :exc:`SubprocessError`. diff --git a/Misc/NEWS.d/next/Library/2023-05-22-18-39-53.gh-issue-104372.7tDRaK.rst b/Misc/NEWS.d/next/Library/2023-05-22-18-39-53.gh-issue-104372.7tDRaK.rst new file mode 100644 index 0000000000000..ea13ec85543ca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-22-18-39-53.gh-issue-104372.7tDRaK.rst @@ -0,0 +1,5 @@ +On Linux where :mod:`subprocess` can use the ``vfork()`` syscall for faster +spawning, prevent the parent process from blocking other threads by dropping +the GIL while it waits for the vfork'ed child process ``exec()`` outcome. +This prevents spawning a binary from a slow filesystem from blocking the +rest of the application. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index ad9daaede430b..072519c91a085 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -537,7 +537,7 @@ reset_signal_handlers(const sigset_t *child_sigmask) * required by POSIX but not supported natively on Linux. Another reason to * avoid this family of functions is that sharing an address space between * processes running with different privileges is inherently insecure. - * See bpo-35823 for further discussion and references. + * See https://bugs.python.org/issue35823 for discussion and references. * * In some C libraries, setrlimit() has the same thread list/signalling * behavior since resource limits were per-thread attributes before @@ -774,6 +774,7 @@ do_fork_exec(char *const exec_array[], pid_t pid; #ifdef VFORK_USABLE + PyThreadState *vfork_tstate_save; if (child_sigmask) { /* These are checked by our caller; verify them in debug builds. */ assert(!call_setuid); @@ -781,8 +782,23 @@ do_fork_exec(char *const exec_array[], assert(!call_setgroups); assert(preexec_fn == Py_None); + /* Drop the GIL so that other threads can continue execution while this + * thread in the parent remains blocked per vfork-semantics on the + * child's exec syscall outcome. Exec does filesystem access which + * can take an arbitrarily long time. This addresses GH-104372. + * + * The vfork'ed child still runs in our address space. Per POSIX it + * must be limited to nothing but exec, but the Linux implementation + * is a little more usable. See the child_exec() comment - The child + * MUST NOT re-acquire the GIL. + */ + vfork_tstate_save = PyEval_SaveThread(); pid = vfork(); - if (pid == -1) { + if (pid != 0) { + // Not in the child process, reacquire the GIL. + PyEval_RestoreThread(vfork_tstate_save); + } + if (pid == (pid_t)-1) { /* If vfork() fails, fall back to using fork(). When it isn't * allowed in a process by the kernel, vfork can return -1 * with errno EINVAL. https://bugs.python.org/issue47151. */ @@ -795,6 +811,7 @@ do_fork_exec(char *const exec_array[], } if (pid != 0) { + // Parent process. return pid; }

1 0

gh-107805: Fix signatures of module-level generated functions in `turtle` (#107807)
by gpshead Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/044b8b3b6a65e6651b161e3badfa5d57c6… commit: 044b8b3b6a65e6651b161e3badfa5d57c666db19 branch: main author: Nikita Sobolev <mail(a)sobolevn.me> committer: gpshead <greg(a)krypto.org> date: 2023-09-01T01:18:15-07:00 summary: gh-107805: Fix signatures of module-level generated functions in `turtle` (#107807) Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst M Lib/test/test_turtle.py M Lib/turtle.py diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 3f9f129a3dd20..14121a590a502 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -461,5 +461,25 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestModuleLevel(unittest.TestCase): + def test_all_signatures(self): + import inspect + + known_signatures = { + 'teleport': + '(x=None, y=None, *, fill_gap: bool = False) -> None', + 'undo': '()', + 'goto': '(x, y=None)', + 'bgcolor': '(*args)', + 'pen': '(pen=None, **pendict)', + } + + for name in known_signatures: + with self.subTest(name=name): + obj = getattr(turtle, name) + sig = inspect.signature(obj) + self.assertEqual(str(sig), known_signatures[name]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/turtle.py b/Lib/turtle.py index e542bc956897e..6119b7a447e7b 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -3920,28 +3920,33 @@ def getmethparlist(ob): function definition and the second is suitable for use in function call. The "self" parameter is not included. """ - defText = callText = "" + orig_sig = inspect.signature(ob) # bit of a hack for methods - turn it into a function # but we drop the "self" param. # Try and build one for Python defined functions - args, varargs, varkw = inspect.getargs(ob.__code__) - items2 = args[1:] - realArgs = args[1:] - defaults = ob.__defaults__ or [] - defaults = ["=%r" % (value,) for value in defaults] - defaults = [""] * (len(realArgs)-len(defaults)) + defaults - items1 = [arg + dflt for arg, dflt in zip(realArgs, defaults)] - if varargs is not None: - items1.append("*" + varargs) - items2.append("*" + varargs) - if varkw is not None: - items1.append("**" + varkw) - items2.append("**" + varkw) - defText = ", ".join(items1) - defText = "(%s)" % defText - callText = ", ".join(items2) - callText = "(%s)" % callText - return defText, callText + func_sig = orig_sig.replace( + parameters=list(orig_sig.parameters.values())[1:], + ) + + call_args = [] + for param in func_sig.parameters.values(): + match param.kind: + case ( + inspect.Parameter.POSITIONAL_ONLY + | inspect.Parameter.POSITIONAL_OR_KEYWORD + ): + call_args.append(param.name) + case inspect.Parameter.VAR_POSITIONAL: + call_args.append(f'*{param.name}') + case inspect.Parameter.KEYWORD_ONLY: + call_args.append(f'{param.name}={param.name}') + case inspect.Parameter.VAR_KEYWORD: + call_args.append(f'**{param.name}') + case _: + raise RuntimeError('Unsupported parameter kind', param.kind) + call_text = f'({', '.join(call_args)})' + + return str(func_sig), call_text def _turtle_docrevise(docstr): """To reduce docstrings from RawTurtle class for functions diff --git a/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst b/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst new file mode 100644 index 0000000000000..263df68f8e5c8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-09-13-49-37.gh-issue-107805.ezem0k.rst @@ -0,0 +1 @@ +Fix signatures of module-level generated functions in :mod:`turtle`.

1 0

gh-106320: Remove private _PyLong_Sign() (#108743)
by vstinner Sept. 1, 2023

Sept. 1, 2023

https://github.com/python/cpython/commit/3edcf743e88b4ac4431d4b0f3a66048628… commit: 3edcf743e88b4ac4431d4b0f3a66048628cf5c70 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-01T09:13:07+02:00 summary: gh-106320: Remove private _PyLong_Sign() (#108743) Move the private _PyLong_Sign() and _PyLong_NumBits() functions to the internal C API (pycore_long.h). Modules/_testcapi/long.c now uses the internal C API. files: M Include/cpython/longobject.h M Include/internal/pycore_long.h M Modules/_io/_iomodule.c M Modules/_testcapi/long.c diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 32c40b083b7ab..57834173490c9 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -4,21 +4,6 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - -/* _PyLong_NumBits. Return the number of bits needed to represent the - absolute value of a long. For example, this returns 1 for 1 and -1, 2 - for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. - v must not be NULL, and must be a normalized long. - (size_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. -*/ -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index c411ac654387c..3c253ed7ff556 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -58,6 +58,23 @@ PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( Py_ssize_t digit_count, digit *digits); +// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. +// v must not be NULL, and must be a normalized long. +// There are no error cases. +// +// Export for '_pickle' shared extension. +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +// _PyLong_NumBits. Return the number of bits needed to represent the +// absolute value of a long. For example, this returns 1 for 1 and -1, 2 +// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. +// v must not be NULL, and must be a normalized long. +// (size_t)-1 is returned and OverflowError set if the true result doesn't +// fit in a size_t. +// +// Export for 'math' shared extension. +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + /* runtime lifecycle */ diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index cc844527da583..173f5b55e5f73 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_long.h" // _PyLong_Sign() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 6b74e0ab8e0d1..c1d2d42a2c434 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -1,5 +1,10 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "parts.h" #include "clinic/long.c.h" +#include "pycore_long.h" // _PyLong_Sign() /*[clinic input] module _testcapi

1 0