Mailman 3 March 2020 - Python-checkins

bpo-40094: Add run_command() to setup.py (GH-19266)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/6b982c22e5fdbfecc24e440515b63f7253… commit: 6b982c22e5fdbfecc24e440515b63f7253f695c4 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-04-01T01:10:07+02:00 summary: bpo-40094: Add run_command() to setup.py (GH-19266) files: M setup.py diff --git a/setup.py b/setup.py index 24ce9a632dd90..3d3e5ac7db03e 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ import sys import sysconfig from glob import glob +from _bootsubprocess import _waitstatus_to_exitcode as waitstatus_to_exitcode try: @@ -95,6 +96,11 @@ def get_platform(): """ +def run_command(cmd): + status = os.system(cmd) + return waitstatus_to_exitcode(status) + + # Set common compiler and linker flags derived from the Makefile, # reserved for building the interpreter and the stdlib modules. # See bpo-21121 and bpo-35257 @@ -176,10 +182,10 @@ def macosx_sdk_root(): os.unlink(tmpfile) except: pass - ret = os.system('%s -E -v - </dev/null 2>%s 1>/dev/null' % (cc, tmpfile)) + ret = run_command('%s -E -v - </dev/null 2>%s 1>/dev/null' % (cc, tmpfile)) in_incdirs = False try: - if ret >> 8 == 0: + if ret == 0: with open(tmpfile) as fp: for line in fp.readlines(): if line.startswith("#include <...>"): @@ -595,11 +601,11 @@ def add_multiarch_paths(self): tmpfile = os.path.join(self.build_temp, 'multiarch') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - ret = os.system( + ret = run_command( '%s -print-multiarch > %s 2> /dev/null' % (cc, tmpfile)) multiarch_path_component = '' try: - if ret >> 8 == 0: + if ret == 0: with open(tmpfile) as fp: multiarch_path_component = fp.readline().strip() finally: @@ -620,11 +626,11 @@ def add_multiarch_paths(self): tmpfile = os.path.join(self.build_temp, 'multiarch') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - ret = os.system( + ret = run_command( 'dpkg-architecture %s -qDEB_HOST_MULTIARCH > %s 2> /dev/null' % (opt, tmpfile)) try: - if ret >> 8 == 0: + if ret == 0: with open(tmpfile) as fp: multiarch_path_component = fp.readline().strip() add_dir_to_list(self.compiler.library_dirs, @@ -639,12 +645,12 @@ def add_cross_compiling_paths(self): tmpfile = os.path.join(self.build_temp, 'ccpaths') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - ret = os.system('%s -E -v - </dev/null 2>%s 1>/dev/null' % (cc, tmpfile)) + ret = run_command('%s -E -v - </dev/null 2>%s 1>/dev/null' % (cc, tmpfile)) is_gcc = False is_clang = False in_incdirs = False try: - if ret >> 8 == 0: + if ret == 0: with open(tmpfile) as fp: for line in fp.readlines(): if line.startswith("gcc version"): @@ -932,14 +938,14 @@ def detect_readline_curses(self): # Determine if readline is already linked against curses or tinfo. if do_readline: if CROSS_COMPILING: - ret = os.system("%s -d %s | grep '(NEEDED)' > %s" \ + ret = run_command("%s -d %s | grep '(NEEDED)' > %s" % (sysconfig.get_config_var('READELF'), do_readline, tmpfile)) elif find_executable('ldd'): - ret = os.system("ldd %s > %s" % (do_readline, tmpfile)) + ret = run_command("ldd %s > %s" % (do_readline, tmpfile)) else: - ret = 256 - if ret >> 8 == 0: + ret = 1 + if ret == 0: with open(tmpfile) as fp: for ln in fp: if 'curses' in ln: @@ -1654,9 +1660,9 @@ def detect_expat_elementtree(self): ] cc = sysconfig.get_config_var('CC').split()[0] - ret = os.system( + ret = run_command( '"%s" -Werror -Wno-unreachable-code -E -xc /dev/null >/dev/null 2>&1' % cc) - if ret >> 8 == 0: + if ret == 0: extra_compile_args.append('-Wno-unreachable-code') self.add(Extension('pyexpat', @@ -1859,9 +1865,9 @@ def detect_tkinter_darwin(self): # Note: cannot use os.popen or subprocess here, that # requires extensions that are not available here. if is_macosx_sdk_path(F): - os.system("file %s/Tk.framework/Tk | grep 'for architecture' > %s"%(os.path.join(sysroot, F[1:]), tmpfile)) + run_command("file %s/Tk.framework/Tk | grep 'for architecture' > %s"%(os.path.join(sysroot, F[1:]), tmpfile)) else: - os.system("file %s/Tk.framework/Tk | grep 'for architecture' > %s"%(F, tmpfile)) + run_command("file %s/Tk.framework/Tk | grep 'for architecture' > %s"%(F, tmpfile)) with open(tmpfile) as fp: detected_archs = []

1 0

bpo-31160: Fix race condition in test_os.PtyTests (GH-19263)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/16d75675d2ad2454f6dfbf333c94e6237d… commit: 16d75675d2ad2454f6dfbf333c94e6237df36018 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-04-01T00:27:18+02:00 summary: bpo-31160: Fix race condition in test_os.PtyTests (GH-19263) bpo-31160, bpo-40094: Wait until the process completes before closing the PTY to prevent sending SIGHUP to the child process. files: M Lib/test/test_builtin.py diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 86e5f1dc0246e..eaada1b50439b 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1846,6 +1846,7 @@ def run_child(self, child, terminal_input): os.close(w) self.skipTest("pty.fork() raised {}".format(e)) raise + if pid == 0: # Child try: @@ -1859,9 +1860,11 @@ def run_child(self, child, terminal_input): finally: # We don't want to return to unittest... os._exit(0) + # Parent os.close(w) os.write(fd, terminal_input) + # Get results from the pipe with open(r, "r") as rpipe: lines = [] @@ -1871,6 +1874,7 @@ def run_child(self, child, terminal_input): # The other end was closed => the child exited break lines.append(line) + # Check the result was got and corresponds to the user's terminal input if len(lines) != 2: # Something went wrong, try to get at stderr @@ -1888,11 +1892,14 @@ def run_child(self, child, terminal_input): child_output = child_output.decode("ascii", "ignore") self.fail("got %d lines in pipe but expected 2, child output was:\n%s" % (len(lines), child_output)) - os.close(fd) - # Wait until the child process completes + # Wait until the child process completes before closing the PTY to + # prevent sending SIGHUP to the child process. support.wait_process(pid, exitcode=0) + # Close the PTY + os.close(fd) + return lines def check_input_tty(self, prompt, terminal_input, stdio_encoding=None):

1 0

bpo-40094: Add _bootsubprocess._waitstatus_to_exitcode (GH-19264)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/40bfdb1594189f3c0238e5d2098dc3abf1… commit: 40bfdb1594189f3c0238e5d2098dc3abf114e200 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T23:45:13+02:00 summary: bpo-40094: Add _bootsubprocess._waitstatus_to_exitcode (GH-19264) * Add _waitstatus_to_exitcode() helper function to _bootsubprocess. * Enhance check_output() error message if the command fails. _bootsubprocess no longer handles WIFSTOPPED() case: it now raises a ValueError. files: M Lib/_bootsubprocess.py diff --git a/Lib/_bootsubprocess.py b/Lib/_bootsubprocess.py index 962301ae1499e..9c1912f315dc9 100644 --- a/Lib/_bootsubprocess.py +++ b/Lib/_bootsubprocess.py @@ -6,6 +6,15 @@ import os +def _waitstatus_to_exitcode(status): + if os.WIFEXITED(status): + return os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + else: + raise ValueError(f"invalid wait status: {status!r}") + + # distutils.spawn used by distutils.command.build_ext # calls subprocess.Popen().wait() class Popen: @@ -27,15 +36,8 @@ def wait(self): os._exit(1) else: # Parent process - pid, status = os.waitpid(pid, 0) - if os.WIFSIGNALED(status): - self.returncode = -os.WTERMSIG(status) - elif os.WIFEXITED(status): - self.returncode = os.WEXITSTATUS(status) - elif os.WIFSTOPPED(status): - self.returncode = -os.WSTOPSIG(status) - else: - raise Exception(f"unknown child process exit status: {status!r}") + _, status = os.waitpid(pid, 0) + self.returncode = _waitstatus_to_exitcode(status) return self.returncode @@ -85,8 +87,10 @@ def check_output(cmd, **kwargs): try: # system() spawns a shell status = os.system(cmd) - if status: - raise ValueError(f"Command {cmd!r} failed with status {status!r}") + exitcode = _waitstatus_to_exitcode(status) + if exitcode: + raise ValueError(f"Command {cmd!r} returned non-zero " + f"exit status {exitcode!r}") try: with open(tmp_filename, "rb") as fp:

1 0

bpo-39943: Clean up marshal.c. (GH-19236)
by Serhiy Storchaka March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/2c003eff8fef430f1876adf88e466bcfcb… commit: 2c003eff8fef430f1876adf88e466bcfcbd0cc9e branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-03-31T23:23:21+03:00 summary: bpo-39943: Clean up marshal.c. (GH-19236) * Add consts. * Remove redundant casts and checks. * Use concrete C API macros. * Avoid raising and silencing OverflowError for ints. files: M Python/marshal.c diff --git a/Python/marshal.c b/Python/marshal.c index 4a23df1dcd865..b4429aea502d3 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -83,7 +83,7 @@ typedef struct { int depth; PyObject *str; char *ptr; - char *end; + const char *end; char *buf; _Py_hashtable_t *hashtable; int version; @@ -114,7 +114,7 @@ w_reserve(WFILE *p, Py_ssize_t needed) } assert(p->str != NULL); pos = p->ptr - p->buf; - size = PyBytes_Size(p->str); + size = PyBytes_GET_SIZE(p->str); if (size > 16*1024*1024) delta = (size >> 3); /* 12.5% overallocation */ else @@ -126,7 +126,7 @@ w_reserve(WFILE *p, Py_ssize_t needed) } size += delta; if (_PyBytes_Resize(&p->str, size) != 0) { - p->ptr = p->buf = p->end = NULL; + p->end = p->ptr = p->buf = NULL; return 0; } else { @@ -138,7 +138,7 @@ w_reserve(WFILE *p, Py_ssize_t needed) } static void -w_string(const char *s, Py_ssize_t n, WFILE *p) +w_string(const void *s, Py_ssize_t n, WFILE *p) { Py_ssize_t m; if (!n || p->ptr == NULL) @@ -194,14 +194,14 @@ w_long(long x, WFILE *p) #endif static void -w_pstring(const char *s, Py_ssize_t n, WFILE *p) +w_pstring(const void *s, Py_ssize_t n, WFILE *p) { W_SIZE(n, p); w_string(s, n, p); } static void -w_short_pstring(const char *s, Py_ssize_t n, WFILE *p) +w_short_pstring(const void *s, Py_ssize_t n, WFILE *p) { w_byte(Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char), p); w_string(s, n, p); @@ -274,21 +274,18 @@ w_float_bin(double v, WFILE *p) p->error = WFERR_UNMARSHALLABLE; return; } - w_string((const char *)buf, 8, p); + w_string(buf, 8, p); } static void w_float_str(double v, WFILE *p) { - int n; char *buf = PyOS_double_to_string(v, 'g', 17, 0, NULL); if (!buf) { p->error = WFERR_NOMEMORY; return; } - n = (int)strlen(buf); - w_byte(n, p); - w_string(buf, n, p); + w_short_pstring(buf, strlen(buf), p); PyMem_Free(buf); } @@ -378,11 +375,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) Py_ssize_t i, n; if (PyLong_CheckExact(v)) { - long x = PyLong_AsLong(v); - if ((x == -1) && PyErr_Occurred()) { - PyLongObject *ob = (PyLongObject *)v; - PyErr_Clear(); - w_PyLong(ob, flag, p); + int overflow; + long x = PyLong_AsLongAndOverflow(v, &overflow); + if (overflow) { + w_PyLong((PyLongObject *)v, flag, p); } else { #if SIZEOF_LONG > 4 @@ -433,7 +429,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) W_TYPE(TYPE_SHORT_ASCII_INTERNED, p); else W_TYPE(TYPE_SHORT_ASCII, p); - w_short_pstring((char *) PyUnicode_1BYTE_DATA(v), + w_short_pstring(PyUnicode_1BYTE_DATA(v), PyUnicode_GET_LENGTH(v), p); } else { @@ -441,7 +437,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) W_TYPE(TYPE_ASCII_INTERNED, p); else W_TYPE(TYPE_ASCII, p); - w_pstring((char *) PyUnicode_1BYTE_DATA(v), + w_pstring(PyUnicode_1BYTE_DATA(v), PyUnicode_GET_LENGTH(v), p); } } @@ -462,7 +458,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } } else if (PyTuple_CheckExact(v)) { - n = PyTuple_Size(v); + n = PyTuple_GET_SIZE(v); if (p->version >= 4 && n < 256) { W_TYPE(TYPE_SMALL_TUPLE, p); w_byte((unsigned char)n, p); @@ -496,34 +492,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object((PyObject *)NULL, p); } else if (PyAnySet_CheckExact(v)) { - PyObject *value, *it; + PyObject *value; + Py_ssize_t pos = 0; + Py_hash_t hash; - if (PyObject_TypeCheck(v, &PySet_Type)) - W_TYPE(TYPE_SET, p); - else + if (PyFrozenSet_CheckExact(v)) W_TYPE(TYPE_FROZENSET, p); - n = PyObject_Size(v); - if (n == -1) { - p->depth--; - p->error = WFERR_UNMARSHALLABLE; - return; - } + else + W_TYPE(TYPE_SET, p); + n = PySet_GET_SIZE(v); W_SIZE(n, p); - it = PyObject_GetIter(v); - if (it == NULL) { - p->depth--; - p->error = WFERR_UNMARSHALLABLE; - return; - } - while ((value = PyIter_Next(it)) != NULL) { + while (_PySet_NextEntry(v, &pos, &value, &hash)) { w_object(value, p); - Py_DECREF(value); - } - Py_DECREF(it); - if (PyErr_Occurred()) { - p->depth--; - p->error = WFERR_UNMARSHALLABLE; - return; } } else if (PyCode_Check(v)) { @@ -638,8 +618,8 @@ typedef struct { FILE *fp; int depth; PyObject *readable; /* Stream-like object being read from */ - char *ptr; - char *end; + const char *ptr; + const char *end; char *buf; Py_ssize_t buf_size; PyObject *refs; /* a list */ @@ -652,7 +632,7 @@ r_string(Py_ssize_t n, RFILE *p) if (p->ptr != NULL) { /* Fast path for loads() */ - char *res = p->ptr; + const char *res = p->ptr; Py_ssize_t left = p->end - p->ptr; if (left < n) { PyErr_SetString(PyExc_EOFError, @@ -1564,8 +1544,8 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) PyObject *result; rf.fp = NULL; rf.readable = NULL; - rf.ptr = (char *)str; - rf.end = (char *)str + len; + rf.ptr = str; + rf.end = str + len; rf.buf = NULL; rf.depth = 0; rf.refs = PyList_New(0); @@ -1587,8 +1567,8 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.str = PyBytes_FromStringAndSize((char *)NULL, 50); if (wf.str == NULL) return NULL; - wf.ptr = wf.buf = PyBytes_AS_STRING((PyBytesObject *)wf.str); - wf.end = wf.ptr + PyBytes_Size(wf.str); + wf.ptr = wf.buf = PyBytes_AS_STRING(wf.str); + wf.end = wf.ptr + PyBytes_GET_SIZE(wf.str); wf.error = WFERR_OK; wf.version = version; if (w_init_refs(&wf, version)) { @@ -1598,13 +1578,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) w_object(x, &wf); w_clear_refs(&wf); if (wf.str != NULL) { - char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str); - if (wf.ptr - base > PY_SSIZE_T_MAX) { - Py_DECREF(wf.str); - PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a bytes object"); - return NULL; - } + const char *base = PyBytes_AS_STRING(wf.str); if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) return NULL; }

1 0

bpo-40094: Enhance threading tests (GH-19260)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/a9f9687a7ce25e7c0c89f88f52db323104… commit: a9f9687a7ce25e7c0c89f88f52db323104668ae0 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T21:49:44+02:00 summary: bpo-40094: Enhance threading tests (GH-19260) * Rewrite test_thread.test_forkinthread() to use support.wait_process() and wait for the child process in the main thread, not in the spawned thread. * test_threading now uses support.wait_process() and checks the child process exit code to detect crashes. files: M Lib/test/test_thread.py M Lib/test/test_threading.py diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 9f4801f47e3aa..77e46f2c2f15a 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -225,30 +225,31 @@ def setUp(self): @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') @support.reap_threads def test_forkinthread(self): - status = "not set" + pid = None - def thread1(): - nonlocal status + def fork_thread(read_fd, write_fd): + nonlocal pid # fork in a thread pid = os.fork() - if pid == 0: - # child - try: - os.close(self.read_fd) - os.write(self.write_fd, b"OK") - finally: - os._exit(0) - else: - # parent - os.close(self.write_fd) - pid, status = os.waitpid(pid, 0) + if pid: + # parent process + return + + # child process + try: + os.close(read_fd) + os.write(write_fd, b"OK") + finally: + os._exit(0) with support.wait_threads_exit(): - thread.start_new_thread(thread1, ()) - self.assertEqual(os.read(self.read_fd, 2), b"OK", - "Unable to fork() in thread") - self.assertEqual(status, 0) + thread.start_new_thread(fork_thread, (self.read_fd, self.write_fd)) + self.assertEqual(os.read(self.read_fd, 2), b"OK") + os.close(self.write_fd) + + self.assertIsNotNone(pid) + support.wait_process(pid, exitcode=0) def tearDown(self): try: diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index da17e1281d986..8a4efd647096a 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -485,9 +485,7 @@ def test_is_alive_after_fork(self): else: t.join() - pid, status = os.waitpid(pid, 0) - self.assertTrue(os.WIFEXITED(status)) - self.assertEqual(10, os.WEXITSTATUS(status)) + support.wait_process(pid, exitcode=10) def test_main_thread(self): main = threading.main_thread() @@ -507,6 +505,7 @@ def f(): def test_main_thread_after_fork(self): code = """if 1: import os, threading + from test import support pid = os.fork() if pid == 0: @@ -515,7 +514,7 @@ def test_main_thread_after_fork(self): print(main.ident == threading.current_thread().ident) print(main.ident == threading.get_ident()) else: - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') @@ -528,6 +527,7 @@ def test_main_thread_after_fork(self): def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: import os, threading, sys + from test import support def f(): pid = os.fork() @@ -540,7 +540,7 @@ def f(): # we have to flush before exit. sys.stdout.flush() else: - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) th = threading.Thread(target=f) th.start() @@ -813,11 +813,15 @@ def test_1_join_on_shutdown(self): def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: + from test import support + childpid = os.fork() if childpid != 0: - os.waitpid(childpid, 0) + # parent process + support.wait_process(childpid, exitcode=0) sys.exit(0) + # child process t = threading.Thread(target=joiningfunc, args=(threading.current_thread(),)) t.start() @@ -832,13 +836,17 @@ def test_3_join_in_forked_from_thread(self): # In the forked process, the main Thread object must be marked as stopped. script = """if 1: + from test import support + main_thread = threading.current_thread() def worker(): childpid = os.fork() if childpid != 0: - os.waitpid(childpid, 0) + # parent process + support.wait_process(childpid, exitcode=0) sys.exit(0) + # child process t = threading.Thread(target=joiningfunc, args=(main_thread,)) print('end of main') @@ -901,9 +909,9 @@ def do_fork_and_wait(): # just fork a child process and wait it pid = os.fork() if pid > 0: - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=50) else: - os._exit(0) + os._exit(50) # start a bunch of threads that will fork() child processes threads = [] @@ -930,12 +938,11 @@ def test_clear_threads_states_after_fork(self): if pid == 0: # check that threads states have been cleared if len(sys._current_frames()) == 1: - os._exit(0) + os._exit(51) else: - os._exit(1) + os._exit(52) else: - _, status = os.waitpid(pid, 0) - self.assertEqual(0, status) + support.wait_process(pid, exitcode=51) for t in threads: t.join()

1 0

bpo-40094: Enhance fork and wait tests (GH-19259)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/27c6231f5827fe17c6cb6f097391931f30… commit: 27c6231f5827fe17c6cb6f097391931f30b511ec branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T21:46:40+02:00 summary: bpo-40094: Enhance fork and wait tests (GH-19259) * test_fork1: remove duplicated wait_impl() method: reuse fork_wait.py implementation instead. * Use exit code different than 0 to ensure that we executed the expected code path. files: M Lib/test/fork_wait.py M Lib/test/test_fork1.py M Lib/test/test_wait3.py M Lib/test/test_wait4.py diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 8c177556a4e33..249b5e9607329 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -43,8 +43,8 @@ def f(self, id): except OSError: pass - def wait_impl(self, cpid): - support.wait_process(cpid, exitcode=0) + def wait_impl(self, cpid, *, exitcode): + support.wait_process(cpid, exitcode=exitcode) def test_wait(self): for i in range(NUM_THREADS): @@ -79,4 +79,4 @@ def test_wait(self): os._exit(n) else: # Parent - self.wait_impl(cpid) + self.wait_impl(cpid, exitcode=0) diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py index ce0a05a541a73..a2f7cfee9cf69 100644 --- a/Lib/test/test_fork1.py +++ b/Lib/test/test_fork1.py @@ -17,19 +17,6 @@ support.get_attribute(os, 'fork') class ForkTest(ForkWait): - def wait_impl(self, cpid): - deadline = time.monotonic() + support.SHORT_TIMEOUT - while time.monotonic() <= deadline: - # waitpid() shouldn't hang, but some of the buildbots seem to hang - # in the forking tests. This is an attempt to fix the problem. - spid, status = os.waitpid(cpid, os.WNOHANG) - if spid == cpid: - break - time.sleep(0.1) - - self.assertEqual(spid, cpid) - self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) - def test_threaded_import_lock_fork(self): """Check fork() in main thread works while a subthread is doing an import""" import_started = threading.Event() @@ -46,6 +33,7 @@ def importer(): t = threading.Thread(target=importer) t.start() import_started.wait() + exitcode = 42 pid = os.fork() try: # PyOS_BeforeFork should have waited for the import to complete @@ -54,7 +42,7 @@ def importer(): if not pid: m = __import__(fake_module_name) if m == complete_module: - os._exit(0) + os._exit(exitcode) else: if support.verbose > 1: print("Child encountered partial module") @@ -64,7 +52,7 @@ def importer(): # Exitcode 1 means the child got a partial module (bad.) No # exitcode (but a hang, which manifests as 'got pid 0') # means the child deadlocked (also bad.) - self.wait_impl(pid) + self.wait_impl(pid, exitcode=exitcode) finally: try: os.kill(pid, signal.SIGKILL) @@ -74,6 +62,7 @@ def importer(): def test_nested_import_lock_fork(self): """Check fork() in main thread works while the main thread is doing an import""" + exitcode = 42 # Issue 9573: this used to trigger RuntimeError in the child process def fork_with_import_lock(level): release = 0 @@ -95,8 +84,8 @@ def fork_with_import_lock(level): os._exit(1) raise if in_child: - os._exit(0) - self.wait_impl(pid) + os._exit(exitcode) + self.wait_impl(pid, exitcode=exitcode) # Check this works with various levels of nested # import in the main thread diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index 2dc63aaffa346..6e06049fbdd69 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -16,7 +16,7 @@ raise unittest.SkipTest("os.wait3 not defined") class Wait3Test(ForkWait): - def wait_impl(self, cpid): + def wait_impl(self, cpid, *, exitcode): # This many iterations can be required, since some previously run # tests (e.g. test_ctypes) could have spawned a lot of children # very quickly. @@ -30,7 +30,8 @@ def wait_impl(self, cpid): time.sleep(0.1) self.assertEqual(spid, cpid) - self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) + self.assertEqual(status, exitcode << 8, + "cause = %d, exit = %d" % (status&0xff, status>>8)) self.assertTrue(rusage) def test_wait3_rusage_initialized(self): diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py index a18607252aa96..6c7ebcb3dd29c 100644 --- a/Lib/test/test_wait4.py +++ b/Lib/test/test_wait4.py @@ -14,7 +14,7 @@ class Wait4Test(ForkWait): - def wait_impl(self, cpid): + def wait_impl(self, cpid, *, exitcode): option = os.WNOHANG if sys.platform.startswith('aix'): # Issue #11185: wait4 is broken on AIX and will always return 0 @@ -29,7 +29,8 @@ def wait_impl(self, cpid): break time.sleep(0.1) self.assertEqual(spid, cpid) - self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) + self.assertEqual(status, exitcode << 8, + "cause = %d, exit = %d" % (status&0xff, status>>8)) self.assertTrue(rusage) def tearDownModule():

1 0

bpo-40094: Add test.support.wait_process() (GH-19254)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/278c1e159c970da6cd6683d18c6211f511… commit: 278c1e159c970da6cd6683d18c6211f5118674cc branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T20:08:12+02:00 summary: bpo-40094: Add test.support.wait_process() (GH-19254) Moreover, the following tests now check the child process exit code: * test_os.PtyTests * test_mailbox.test_lock_conflict() * test_tempfile.test_process_awareness() * test_uuid.testIssue8621() * multiprocessing resource tracker tests files: A Misc/NEWS.d/next/Tests/2020-03-31-18-57-52.bpo-40094.m3fTJe.rst M Doc/library/test.rst M Lib/test/_test_multiprocessing.py M Lib/test/fork_wait.py M Lib/test/support/__init__.py M Lib/test/test_builtin.py M Lib/test/test_logging.py M Lib/test/test_mailbox.py M Lib/test/test_os.py M Lib/test/test_platform.py M Lib/test/test_posix.py M Lib/test/test_random.py M Lib/test/test_socketserver.py M Lib/test/test_ssl.py M Lib/test/test_subprocess.py M Lib/test/test_support.py M Lib/test/test_tempfile.py M Lib/test/test_tracemalloc.py M Lib/test/test_uuid.py diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 54ad620d7dae9..c33465d758d57 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -825,6 +825,21 @@ The :mod:`test.support` module defines the following functions: target of the "as" clause, if there is one. +.. function:: wait_process(pid, *, exitcode, timeout=None) + + Wait until process *pid* completes and check that the process exit code is + *exitcode*. + + Raise an :exc:`AssertionError` if the process exit code is not equal to + *exitcode*. + + If the process runs longer than *timeout* seconds (:data:`SHORT_TIMEOUT` by + default), kill the process and raise an :exc:`AssertionError`. The timeout + feature is not available on Windows. + + .. versionadded:: 3.9 + + .. function:: wait_threads_exit(timeout=60.0) Context manager to wait until all threads created in the ``with`` statement diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4a87b1761f9ef..d00e928c17790 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5124,7 +5124,7 @@ def check_resource_tracker_death(self, signum, should_die): pid = _resource_tracker._pid if pid is not None: os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=-signal.SIGKILL) with warnings.catch_warnings(): warnings.simplefilter("ignore") _resource_tracker.ensure_running() diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index f6bbffe1e7fdc..8c177556a4e33 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -44,16 +44,7 @@ def f(self, id): pass def wait_impl(self, cpid): - for i in range(10): - # waitpid() shouldn't hang, but some of the buildbots seem to hang - # in the forking tests. This is an attempt to fix the problem. - spid, status = os.waitpid(cpid, os.WNOHANG) - if spid == cpid: - break - time.sleep(2 * SHORTSLEEP) - - self.assertEqual(spid, cpid) - self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) + support.wait_process(cpid, exitcode=0) def test_wait(self): for i in range(NUM_THREADS): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 259c7069bfc0a..5b9aebbda7971 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3400,3 +3400,62 @@ def __exit__(self, *exc_info): del self.exc_value del self.exc_traceback del self.thread + + +def wait_process(pid, *, exitcode, timeout=None): + """ + Wait until process pid completes and check that the process exit code is + exitcode. + + Raise an AssertionError if the process exit code is not equal to exitcode. + + If the process runs longer than timeout seconds (SHORT_TIMEOUT by default), + kill the process (if signal.SIGKILL is available) and raise an + AssertionError. The timeout feature is not available on Windows. + """ + if os.name != "nt": + if timeout is None: + timeout = SHORT_TIMEOUT + t0 = time.monotonic() + deadline = t0 + timeout + sleep = 0.001 + max_sleep = 0.1 + while True: + pid2, status = os.waitpid(pid, os.WNOHANG) + if pid2 != 0: + break + # process is still running + + dt = time.monotonic() - t0 + if dt > SHORT_TIMEOUT: + try: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except OSError: + # Ignore errors like ChildProcessError or PermissionError + pass + + raise AssertionError(f"process {pid} is still running " + f"after {dt:.1f} seconds") + + sleep = min(sleep * 2, max_sleep) + time.sleep(sleep) + + if os.WIFEXITED(status): + exitcode2 = os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + exitcode2 = -os.WTERMSIG(status) + else: + raise ValueError(f"invalid wait status: {status!r}") + else: + # Windows implementation + pid2, status = os.waitpid(pid, 0) + exitcode2 = (status >> 8) + + if exitcode2 != exitcode: + raise AssertionError(f"process {pid} exited with code {exitcode2}, " + f"but exit code {exitcode} is expected") + + # sanity check: it should not fail in practice + if pid2 != pid: + raise AssertionError(f"pid {pid2} != pid {pid}") diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 1e9012e7e5ed2..86e5f1dc0246e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -25,6 +25,7 @@ from textwrap import dedent from types import AsyncGeneratorType, FunctionType from operator import neg +from test import support from test.support import ( EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink, maybe_get_event_loop_policy) @@ -1890,7 +1891,7 @@ def run_child(self, child, terminal_input): os.close(fd) # Wait until the child process completes - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) return lines diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e223522cc7ecc..2ad3c5c208583 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -727,30 +727,19 @@ def lock_holder_thread_fn(): locks_held__ready_to_fork.wait() pid = os.fork() - if pid == 0: # Child. + if pid == 0: + # Child process try: test_logger.info(r'Child process did not deadlock. \o/') finally: os._exit(0) - else: # Parent. + else: + # Parent process test_logger.info(r'Parent process returned from fork. \o/') fork_happened__release_locks_and_end_thread.set() lock_holder_thread.join() - start_time = time.monotonic() - while True: - test_logger.debug('Waiting for child process.') - waited_pid, status = os.waitpid(pid, os.WNOHANG) - if waited_pid == pid: - break # child process exited. - if time.monotonic() - start_time > 7: - break # so long? implies child deadlock. - time.sleep(0.05) - test_logger.debug('Done waiting.') - if waited_pid != pid: - os.kill(pid, signal.SIGKILL) - waited_pid, status = os.waitpid(pid, 0) - self.fail("child process deadlocked.") - self.assertEqual(status, 0, msg="child process error") + + support.wait_process(pid, exitcode=0) class BadStream(object): diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 36a265390e985..fdda1d11d3307 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1092,7 +1092,7 @@ def test_lock_conflict(self): # Signal the child it can now release the lock and exit. p.send(b'p') # Wait for child to exit. Locking should now succeed. - exited_pid, status = os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) self._box.lock() self._box.unlock() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9c965444f04b2..be85616ff88bf 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2792,8 +2792,7 @@ def test_waitpid(self): args = [sys.executable, '-c', 'pass'] # Add an implicit test for PyUnicode_FSConverter(). pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) - status = os.waitpid(pid, 0) - self.assertEqual(status, (pid, 0)) + support.wait_process(pid, exitcode=0) class SpawnTests(unittest.TestCase): @@ -2877,14 +2876,7 @@ def test_spawnvpe(self): def test_nowait(self): args = self.create_args() pid = os.spawnv(os.P_NOWAIT, args[0], args) - result = os.waitpid(pid, 0) - self.assertEqual(result[0], pid) - status = result[1] - if hasattr(os, 'WIFEXITED'): - self.assertTrue(os.WIFEXITED(status)) - self.assertEqual(os.WEXITSTATUS(status), self.exitcode) - else: - self.assertEqual(status, self.exitcode << 8) + support.wait_process(pid, exitcode=self.exitcode) @requires_os_func('spawnve') def test_spawnve_bytes(self): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 3084663a8fadd..f167fb1e7b9bf 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -236,9 +236,7 @@ def test_mac_ver_with_fork(self): else: # parent - cpid, sts = os.waitpid(pid, 0) - self.assertEqual(cpid, pid) - self.assertEqual(sts, 0) + support.wait_process(pid, exitcode=0) def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index fad26d88be2f3..be121ae463dbb 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -37,6 +37,7 @@ def _supports_sched(): requires_sched = unittest.skipUnless(_supports_sched(), 'requires POSIX scheduler API') + class PosixTester(unittest.TestCase): def setUp(self): @@ -180,7 +181,6 @@ def test_truncate(self): @unittest.skipUnless(getattr(os, 'execve', None) in os.supports_fd, "test needs execve() to support the fd parameter") @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") - @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_fexecve(self): fp = os.open(sys.executable, os.O_RDONLY) try: @@ -189,7 +189,7 @@ def test_fexecve(self): os.chdir(os.path.split(sys.executable)[0]) posix.execve(fp, [sys.executable, '-c', 'pass'], os.environ) else: - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) finally: os.close(fp) @@ -1539,7 +1539,7 @@ def test_returns_pid(self): """ args = self.python_args('-c', script) pid = self.spawn_func(args[0], args, os.environ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) with open(pidfile) as f: self.assertEqual(f.read(), str(pid)) @@ -1569,7 +1569,7 @@ def test_specify_environment(self): args = self.python_args('-c', script) pid = self.spawn_func(args[0], args, {**os.environ, 'foo': 'bar'}) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) with open(envfile) as f: self.assertEqual(f.read(), 'bar') @@ -1580,7 +1580,7 @@ def test_none_file_actions(self): os.environ, file_actions=None ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_empty_file_actions(self): pid = self.spawn_func( @@ -1589,7 +1589,7 @@ def test_empty_file_actions(self): os.environ, file_actions=[] ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_resetids_explicit_default(self): pid = self.spawn_func( @@ -1598,7 +1598,7 @@ def test_resetids_explicit_default(self): os.environ, resetids=False ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_resetids(self): pid = self.spawn_func( @@ -1607,7 +1607,7 @@ def test_resetids(self): os.environ, resetids=True ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_resetids_wrong_type(self): with self.assertRaises(TypeError): @@ -1622,7 +1622,7 @@ def test_setpgroup(self): os.environ, setpgroup=os.getpgrp() ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_setpgroup_wrong_type(self): with self.assertRaises(TypeError): @@ -1643,7 +1643,7 @@ def test_setsigmask(self): os.environ, setsigmask=[signal.SIGUSR1] ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_setsigmask_wrong_type(self): with self.assertRaises(TypeError): @@ -1684,7 +1684,8 @@ def test_setsid(self): finally: os.close(wfd) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) + output = os.read(rfd, 100) child_sid = int(output) parent_sid = os.getsid(os.getpid()) @@ -1707,10 +1708,7 @@ def test_setsigdef(self): finally: signal.signal(signal.SIGUSR1, original_handler) - pid2, status = os.waitpid(pid, 0) - self.assertEqual(pid2, pid) - self.assertTrue(os.WIFSIGNALED(status), status) - self.assertEqual(os.WTERMSIG(status), signal.SIGUSR1) + support.wait_process(pid, exitcode=-signal.SIGUSR1) def test_setsigdef_wrong_type(self): with self.assertRaises(TypeError): @@ -1744,7 +1742,7 @@ def test_setscheduler_only_param(self): os.environ, scheduler=(None, os.sched_param(priority)) ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), @@ -1764,7 +1762,7 @@ def test_setscheduler_with_policy(self): os.environ, scheduler=(policy, os.sched_param(priority)) ) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_multiple_file_actions(self): file_actions = [ @@ -1776,7 +1774,7 @@ def test_multiple_file_actions(self): self.NOOP_PROGRAM, os.environ, file_actions=file_actions) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) def test_bad_file_actions(self): args = self.NOOP_PROGRAM @@ -1822,7 +1820,8 @@ def test_open_file(self): args = self.python_args('-c', script) pid = self.spawn_func(args[0], args, os.environ, file_actions=file_actions) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + support.wait_process(pid, exitcode=0) with open(outfile) as f: self.assertEqual(f.read(), 'hello') @@ -1840,7 +1839,8 @@ def test_close_file(self): args = self.python_args('-c', script) pid = self.spawn_func(args[0], args, os.environ, file_actions=[(os.POSIX_SPAWN_CLOSE, 0)]) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + + support.wait_process(pid, exitcode=0) with open(closefile) as f: self.assertEqual(f.read(), 'is closed %d' % errno.EBADF) @@ -1858,7 +1858,7 @@ def test_dup2(self): args = self.python_args('-c', script) pid = self.spawn_func(args[0], args, os.environ, file_actions=file_actions) - self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + support.wait_process(pid, exitcode=0) with open(dupfile) as f: self.assertEqual(f.read(), 'hello') @@ -1890,13 +1890,12 @@ def test_posix_spawnp(self): spawn_args = (program, '-I', '-S', '-c', 'pass') code = textwrap.dedent(""" import os + from test import support + args = %a pid = os.posix_spawnp(args[0], args, os.environ) - pid2, status = os.waitpid(pid, 0) - if pid2 != pid: - raise Exception(f"pid {pid2} != {pid}") - if status != 0: - raise Exception(f"status {status} != 0") + + support.wait_process(pid, exitcode=0) """ % (spawn_args,)) # Use a subprocess to test os.posix_spawnp() with a modified PATH diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index c147105376199..548af706dbee2 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1103,8 +1103,7 @@ def test_after_fork(self): child_val = eval(f.read()) self.assertNotEqual(val, child_val) - pid, status = os.waitpid(pid, 0) - self.assertEqual(status, 0) + support.wait_process(pid, exitcode=0) if __name__ == "__main__": diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 85382a013136d..f818df0b22f6a 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -65,9 +65,7 @@ def simple_subprocess(testcase): except: raise finally: - pid2, status = os.waitpid(pid, 0) - testcase.assertEqual(pid2, pid) - testcase.assertEqual(72 << 8, status) + test.support.wait_process(pid, exitcode=72) class SocketServerTest(unittest.TestCase): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 0093a49115d26..4184665b2b158 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -408,8 +408,7 @@ def test_random_fork(self): else: os.close(wfd) self.addCleanup(os.close, rfd) - _, status = os.waitpid(pid, 0) - self.assertEqual(status, 0) + support.wait_process(pid, exitcode=0) child_random = os.read(rfd, 16) self.assertEqual(len(child_random), 16) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 868f279839455..7cf31e1f0921d 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3114,12 +3114,10 @@ def test_stopped(self): proc = subprocess.Popen(args) # Wait until the real process completes to avoid zombie process - pid = proc.pid - pid, status = os.waitpid(pid, 0) - self.assertEqual(status, 0) + support.wait_process(proc.pid, exitcode=0) status = _testcapi.W_STOPCODE(3) - with mock.patch('subprocess.os.waitpid', return_value=(pid, status)): + with mock.patch('subprocess.os.waitpid', return_value=(proc.pid, status)): returncode = proc.wait() self.assertEqual(returncode, -3) @@ -3130,10 +3128,7 @@ def test_send_signal_race(self): proc = subprocess.Popen(ZERO_RETURN_CMD) # wait until the process completes without using the Popen APIs. - pid, status = os.waitpid(proc.pid, 0) - self.assertEqual(pid, proc.pid) - self.assertTrue(os.WIFEXITED(status), status) - self.assertEqual(os.WEXITSTATUS(status), 0) + support.wait_process(proc.pid, exitcode=0) # returncode is still None but the process completed. self.assertIsNone(proc.returncode) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 175f7c845fe8c..99a4cad2bb887 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -176,13 +176,10 @@ def test_temp_dir__forked_child(self): with support.temp_cwd() as temp_path: pid = os.fork() if pid != 0: - # parent process (child has pid == 0) + # parent process # wait for the child to terminate - (pid, status) = os.waitpid(pid, 0) - if status != 0: - raise AssertionError(f"Child process failed with exit " - f"status indication 0x{status:x}.") + support.wait_process(pid, exitcode=0) # Make sure that temp_path is still present. When the child # process leaves the 'temp_cwd'-context, the __exit__()- diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 5fe9506b0b7ba..524ab7c6d766f 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -200,15 +200,7 @@ def test_process_awareness(self): child_value = os.read(read_fd, len(parent_value)).decode("ascii") finally: if pid: - # best effort to ensure the process can't bleed out - # via any bugs above - try: - os.kill(pid, signal.SIGKILL) - except OSError: - pass - - # Read the process exit status to avoid zombie process - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) os.close(read_fd) os.close(write_fd) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 7283d9c31db4e..635a9d3981605 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -314,10 +314,7 @@ def test_fork(self): finally: os._exit(exitcode) else: - pid2, status = os.waitpid(pid, 0) - self.assertTrue(os.WIFEXITED(status)) - exitcode = os.WEXITSTATUS(status) - self.assertEqual(exitcode, 0) + support.wait_process(pid, exitcode=0) class TestSnapshot(unittest.TestCase): diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 27fc56d226c08..0b267f4a97861 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -657,7 +657,7 @@ def testIssue8621(self): os.close(fds[1]) self.addCleanup(os.close, fds[0]) parent_value = self.uuid.uuid4().hex - os.waitpid(pid, 0) + support.wait_process(pid, exitcode=0) child_value = os.read(fds[0], 100).decode('latin-1') self.assertNotEqual(parent_value, child_value) diff --git a/Misc/NEWS.d/next/Tests/2020-03-31-18-57-52.bpo-40094.m3fTJe.rst b/Misc/NEWS.d/next/Tests/2020-03-31-18-57-52.bpo-40094.m3fTJe.rst new file mode 100644 index 0000000000000..cae001bcb209e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-03-31-18-57-52.bpo-40094.m3fTJe.rst @@ -0,0 +1 @@ +Add :func:`test.support.wait_process` function.

1 0

Document most common signals (GH-19245) (GH-19258)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/ad8e56d0941838240b24b7dcab6e082550… commit: ad8e56d0941838240b24b7dcab6e082550191022 branch: 3.7 author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T19:44:34+02:00 summary: Document most common signals (GH-19245) (GH-19258) Document individual signals (only the most common signals): description, default action, availability. (cherry picked from commit 400e1dbcad93061f1f7ab4735202daaa5e731507) files: M Doc/library/signal.rst diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 75008118b7ca2..7f48f8c25f4e0 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -91,6 +91,110 @@ The variables defined in the :mod:`signal` module are: signal. +.. data:: SIGABRT + + Abort signal from :manpage:`abort(3)`. + +.. data:: SIGALRM + + Timer signal from :manpage:`alarm(2)`. + + .. availability:: Unix. + +.. data:: SIGBREAK + + Interrupt from keyboard (CTRL + BREAK). + + .. availability:: Windows. + +.. data:: SIGBUS + + Bus error (bad memory access). + + .. availability:: Unix. + +.. data:: SIGCHLD + + Child process stopped or terminated. + + .. availability:: Windows. + +.. data:: SIGCLD + + Alias to :data:`SIGCHLD`. + +.. data:: SIGCONT + + Continue the process if it is currently stopped + + .. availability:: Unix. + +.. data:: SIGFPE + + Floating-point exception. For example, division by zero. + + .. seealso:: + :exc:`ZeroDivisionError` is raised when the second argument of a division + or modulo operation is zero. + +.. data:: SIGHUP + + Hangup detected on controlling terminal or death of controlling process. + + .. availability:: Unix. + +.. data:: SIGILL + + Illegal instruction. + +.. data:: SIGINT + + Interrupt from keyboard (CTRL + C). + + Default action is to raise :exc:`KeyboardInterrupt`. + +.. data:: SIGKILL + + Kill signal. + + It cannot be caught, blocked, or ignored. + + .. availability:: Unix. + +.. data:: SIGPIPE + + Broken pipe: write to pipe with no readers. + + Default action is to ignore the signal. + + .. availability:: Unix. + +.. data:: SIGSEGV + + Segmentation fault: invalid memory reference. + +.. data:: SIGTERM + + Termination signal. + +.. data:: SIGUSR1 + + User-defined signal 1. + + .. availability:: Unix. + +.. data:: SIGUSR2 + + User-defined signal 2. + + .. availability:: Unix. + +.. data:: SIGWINCH + + Window resize signal. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal @@ -270,6 +374,8 @@ The :mod:`signal` module defines the following functions: For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the signal mask of the calling thread. + :data:`SIGKILL` and :data:`SIGSTOP` cannot be blocked. + .. availability:: Unix. See the man page :manpage:`sigprocmask(3)` and :manpage:`pthread_sigmask(3)` for further information.

1 0

Document most common signals (GH-19245) (GH-19257)
by Victor Stinner March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/40e1b04e389f2f6d4d31079d5622fc27af… commit: 40e1b04e389f2f6d4d31079d5622fc27af6ebed7 branch: 3.8 author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-03-31T19:44:28+02:00 summary: Document most common signals (GH-19245) (GH-19257) Document individual signals (only the most common signals): description, default action, availability. (cherry picked from commit 400e1dbcad93061f1f7ab4735202daaa5e731507) files: M Doc/library/signal.rst M Python/pylifecycle.c diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 932201b0e9d7d..5488f4a1f9685 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -91,6 +91,110 @@ The variables defined in the :mod:`signal` module are: signal. +.. data:: SIGABRT + + Abort signal from :manpage:`abort(3)`. + +.. data:: SIGALRM + + Timer signal from :manpage:`alarm(2)`. + + .. availability:: Unix. + +.. data:: SIGBREAK + + Interrupt from keyboard (CTRL + BREAK). + + .. availability:: Windows. + +.. data:: SIGBUS + + Bus error (bad memory access). + + .. availability:: Unix. + +.. data:: SIGCHLD + + Child process stopped or terminated. + + .. availability:: Windows. + +.. data:: SIGCLD + + Alias to :data:`SIGCHLD`. + +.. data:: SIGCONT + + Continue the process if it is currently stopped + + .. availability:: Unix. + +.. data:: SIGFPE + + Floating-point exception. For example, division by zero. + + .. seealso:: + :exc:`ZeroDivisionError` is raised when the second argument of a division + or modulo operation is zero. + +.. data:: SIGHUP + + Hangup detected on controlling terminal or death of controlling process. + + .. availability:: Unix. + +.. data:: SIGILL + + Illegal instruction. + +.. data:: SIGINT + + Interrupt from keyboard (CTRL + C). + + Default action is to raise :exc:`KeyboardInterrupt`. + +.. data:: SIGKILL + + Kill signal. + + It cannot be caught, blocked, or ignored. + + .. availability:: Unix. + +.. data:: SIGPIPE + + Broken pipe: write to pipe with no readers. + + Default action is to ignore the signal. + + .. availability:: Unix. + +.. data:: SIGSEGV + + Segmentation fault: invalid memory reference. + +.. data:: SIGTERM + + Termination signal. + +.. data:: SIGUSR1 + + User-defined signal 1. + + .. availability:: Unix. + +.. data:: SIGUSR2 + + User-defined signal 2. + + .. availability:: Unix. + +.. data:: SIGWINCH + + Window resize signal. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal @@ -297,6 +401,8 @@ The :mod:`signal` module defines the following functions: For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the signal mask of the calling thread. + :data:`SIGKILL` and :data:`SIGSTOP` cannot be blocked. + .. availability:: Unix. See the man page :manpage:`sigprocmask(3)` and :manpage:`pthread_sigmask(3)` for further information. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index feb9285239240..27cebf33da544 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2311,7 +2311,7 @@ init_signals(void) #ifdef SIGXFSZ PyOS_setsig(SIGXFSZ, SIG_IGN); #endif - PyOS_InitInterrupts(); /* May imply initsignal() */ + PyOS_InitInterrupts(); /* May imply init_signals() */ if (PyErr_Occurred()) { return _PyStatus_ERR("can't import signal"); }

1 0

bpo-40019: Skip test_gdb if Python was optimized (GH-19081)
by Miss Islington (bot) March 31, 2020

March 31, 2020

https://github.com/python/cpython/commit/a764a1cc663708361300cdf88fcf697633… commit: a764a1cc663708361300cdf88fcf697633142319 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-03-31T10:28:35-07:00 summary: bpo-40019: Skip test_gdb if Python was optimized (GH-19081) test_gdb now skips tests if it detects that gdb failed to read debug information because the Python binary is optimized. (cherry picked from commit 7bf069b6110278102c8f4719975a5eb5a5af25f9) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/Tests/2020-03-20-00-30-36.bpo-40019.zOqHpQ.rst M Lib/test/test_gdb.py M Tools/gdb/libpython.py diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 1a9975fb84d1e..0131e4d0b0f72 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -227,6 +227,15 @@ def get_stack_trace(self, source=None, script=None, " because the Program Counter is" " not present") + # bpo-40019: Skip the test if gdb failed to read debug information + # because the Python binary is optimized. + for pattern in ( + '(frame information optimized out)', + 'Unable to read information on python frame', + ): + if pattern in out: + raise unittest.SkipTest(f"{pattern!r} found in gdb output") + return out def get_gdb_repr(self, source, diff --git a/Misc/NEWS.d/next/Tests/2020-03-20-00-30-36.bpo-40019.zOqHpQ.rst b/Misc/NEWS.d/next/Tests/2020-03-20-00-30-36.bpo-40019.zOqHpQ.rst new file mode 100644 index 0000000000000..a9d0b3970ae53 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2020-03-20-00-30-36.bpo-40019.zOqHpQ.rst @@ -0,0 +1,2 @@ +test_gdb now skips tests if it detects that gdb failed to read debug +information because the Python binary is optimized. diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index feec789e6ade9..86ace3796218f 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -99,6 +99,8 @@ def _sizeof_void_p(): ENCODING = locale.getpreferredencoding() +FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' +UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' class NullPyObjectPtr(RuntimeError): @@ -918,7 +920,7 @@ def get_var_by_name(self, name): def filename(self): '''Get the path of the current Python source file, as a string''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT return self.co_filename.proxyval(set()) def current_line_num(self): @@ -949,7 +951,7 @@ def current_line(self): '''Get the text of the current source line as a string, with a trailing newline character''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT lineno = self.current_line_num() if lineno is None: @@ -970,7 +972,7 @@ def current_line(self): def write_repr(self, out, visited): if self.is_optimized_out(): - out.write('(frame information optimized out)') + out.write(FRAME_INFO_OPTIMIZED_OUT) return lineno = self.current_line_num() lineno = str(lineno) if lineno is not None else "?" @@ -993,7 +995,7 @@ def write_repr(self, out, visited): def print_traceback(self): if self.is_optimized_out(): - sys.stdout.write(' (frame information optimized out)\n') + sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) return visited = set() lineno = self.current_line_num() @@ -1744,7 +1746,7 @@ def invoke(self, args, from_tty): pyop = frame.get_pyop() if not pyop or pyop.is_optimized_out(): - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return filename = pyop.filename() @@ -1904,7 +1906,7 @@ def invoke(self, args, from_tty): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return pyop_var, scope = pyop_frame.get_var_by_name(name) @@ -1938,7 +1940,7 @@ def invoke(self, args, from_tty): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return for pyop_name, pyop_value in pyop_frame.iter_locals():

1 0