Mailman 3 August 2017 - Python-checkins

[2.7] bpo-31160: Backport reap_children fixes from master to 2.7 (#3063)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/1247e2cda514d7a73187e0b53ec8c35d87… commit: 1247e2cda514d7a73187e0b53ec8c35d87a34a84 branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T16:45:38+02:00 summary: [2.7] bpo-31160: Backport reap_children fixes from master to 2.7 (#3063) * bpo-31160: regrtest now reaps child processes (#3044) Add a post_test_cleanup() function which currently only calls support.reap_children(). (cherry picked from commit e3510d74aacc477c30f42f2b941d69689bbc478e) * bpo-31160: test_tempfile: Fix reap_children() warning (#3056) TestRandomNameSequence.test_process_awareness() now calls os.waitpid() to avoid leaking a zombie process. (cherry picked from commit 6c8c2943d996b59a48d331f61f22cbe72933910e) files: M Lib/test/regrtest.py M Lib/test/test_tempfile.py diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 7a48475ee8a..7096de8469a 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1172,6 +1172,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False +def post_test_cleanup(): + test_support.reap_children() + def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=None): test_support.unload(test) if verbose: @@ -1205,6 +1208,7 @@ def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=Non refleak = dash_R(the_module, test, indirect_test, huntrleaks) test_time = time.time() - start_time + post_test_cleanup() finally: sys.stdout = save_stdout except test_support.ResourceDenied, msg: diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 5c85cc9bd30..5c111a29ca6 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -141,12 +141,15 @@ def test_process_awareness(self): try: pid = os.fork() if not pid: + # child process os.close(read_fd) os.write(write_fd, next(self.r).encode("ascii")) os.close(write_fd) # bypass the normal exit handlers- leave those to # the parent. os._exit(0) + + # parent process parent_value = next(self.r) child_value = os.read(read_fd, len(parent_value)).decode("ascii") finally: @@ -157,6 +160,10 @@ def test_process_awareness(self): os.kill(pid, signal.SIGKILL) except EnvironmentError: pass + + # Read the process exit status to avoid zombie process + os.waitpid(pid, 0) + os.close(read_fd) os.close(write_fd) self.assertNotEqual(child_value, parent_value)

1 0

regrtest: count also env changed as failures in progress (#3061)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/b97d99cd7f9546493ecc995f003f7b0ba4… commit: b97d99cd7f9546493ecc995f003f7b0ba4ba3a17 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T16:19:03+02:00 summary: regrtest: count also env changed as failures in progress (#3061) files: M Lib/test/libregrtest/main.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index bc8155bbff2..0b48bebb0d7 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -132,8 +132,9 @@ def display_progress(self, test_index, test): # "[ 51/405/1] test_tcl passed" line = f"{test_index:{self.test_count_width}}{self.test_count}" - if self.bad and not self.ns.pgo: - line = f"{line}/{len(self.bad)}" + fails = len(self.bad) + len(self.environment_changed) + if fails and not self.ns.pgo: + line = f"{line}/{fails}" line = f"[{line}] {test}" # add the system load prefix: "load avg: 1.80 "

1 0

PCbuild: downgrade pcbuild.sln to support VS 2010 (#3031)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/693790817a93701093ef93333be7abbb0d… commit: 693790817a93701093ef93333be7abbb0d6d4309 branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T16:11:29+02:00 summary: PCbuild: downgrade pcbuild.sln to support VS 2010 (#3031) bpo-31133: Downgrade PCbuild/pcbuild.sln from format version 12.00 to 11.00 to support Visual Studio 2010 and newer, not only VS 2013 and newer. files: M PCbuild/pcbuild.sln diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 069116061f7..e03861bbf99 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,4 +1,4 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 11.00 # Visual Studio 2010 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject

1 0

[3.6] bpo-31160: Backport reap_children() fixes from master to 3.6 (#3060)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/719a15b32587de6c4add4385ee3f84a207… commit: 719a15b32587de6c4add4385ee3f84a20711340f branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T16:02:00+02:00 summary: [3.6] bpo-31160: Backport reap_children() fixes from master to 3.6 (#3060) * bpo-31160: Fix test_builtin for zombie process (#3043) PtyTests.run_child() now calls os.waitpid() to read the exit status of the child process to avoid creating zombie process and leaking processes in the background. (cherry picked from commit 4baca1b0f7325032598cd38e7ceffc79b616d255) * bpo-31160: regrtest now reaps child processes (#3044) Add a post_test_cleanup() function which currently only calls support.reap_children(). (cherry picked from commit e3510d74aacc477c30f42f2b941d69689bbc478e) * bpo-31160: test_builtin: don't check waitpid() status (#3050) (cherry picked from commit 3ca9f50f96cfa5c1b7aa56639042531b57f07fbb) * bpo-31160: test_tempfile: Fix reap_children() warning (#3056) TestRandomNameSequence.test_process_awareness() now calls os.waitpid() to avoid leaking a zombie process. (cherry picked from commit 6c8c2943d996b59a48d331f61f22cbe72933910e) files: M Lib/test/libregrtest/runtest.py M Lib/test/test_builtin.py M Lib/test/test_tempfile.py diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fda4ca1ebca..d716b83e235 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -143,6 +143,10 @@ def runtest(ns, test): runtest.stringio = None +def post_test_cleanup(): + support.reap_children() + + def runtest_inner(ns, test, display_failure=True): support.unload(test) @@ -170,6 +174,7 @@ def test_runner(): if ns.huntrleaks: refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) test_time = time.time() - start_time + post_test_cleanup() except support.ResourceDenied as msg: if not ns.quiet and not ns.pgo: print(test, "skipped --", msg, flush=True) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 7a4b7eb9176..49c4a53c052 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1560,6 +1560,10 @@ def run_child(self, child, terminal_input): self.fail("got %d lines in pipe but expected 2, child output was:\n%s" % (len(lines), child_output)) os.close(fd) + + # Wait until the child process completes + os.waitpid(pid, 0) + return lines def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index d0cf04b0cb6..710756bde64 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -78,7 +78,6 @@ def setUp(self): def tearDown(self): self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] @@ -184,12 +183,15 @@ def test_process_awareness(self): try: pid = os.fork() if not pid: + # child process os.close(read_fd) os.write(write_fd, next(self.r).encode("ascii")) os.close(write_fd) # bypass the normal exit handlers- leave those to # the parent. os._exit(0) + + # parent process parent_value = next(self.r) child_value = os.read(read_fd, len(parent_value)).decode("ascii") finally: @@ -200,6 +202,10 @@ def test_process_awareness(self): os.kill(pid, signal.SIGKILL) except OSError: pass + + # Read the process exit status to avoid zombie process + os.waitpid(pid, 0) + os.close(read_fd) os.close(write_fd) self.assertNotEqual(child_value, parent_value)

1 0

Enhance support.reap_children() (#3036)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/b5011479808b80545bdd1246725fc79406… commit: b5011479808b80545bdd1246725fc7940691b084 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T16:01:47+02:00 summary: Enhance support.reap_children() (#3036) * reap_children() now sets environment_altered to True to detect bugs using python3 -m test --fail-env-changed * Replace bare "except:" with "except OSError:" in reap_children() * Write an unit test for reap_children() using a timeout of 60 seconds files: M Lib/test/support/__init__.py M Lib/test/test_support.py diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 7a4671c1b2b..0235498e2a9 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2073,26 +2073,35 @@ def decorator(*args): threading_cleanup(*key) return decorator + def reap_children(): """Use this function at the end of test_main() whenever sub-processes are started. This will help ensure that no extra children (zombies) stick around to hog resources and create problems when looking for refleaks. """ + global environment_altered + + # Need os.waitpid(-1, os.WNOHANG): Windows is not supported + if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')): + return + # Reap all our dead child processes so we don't leave zombies around. # These hog resources and might be causing some of the buildbots to die. - if hasattr(os, 'waitpid'): - any_process = -1 - while True: - try: - # This will raise an exception on Windows. That's ok. - pid, status = os.waitpid(any_process, os.WNOHANG) - if pid == 0: - break - print("Warning -- reap_children() reaped child process %s" - % pid, file=sys.stderr) - except: - break + while True: + try: + # Read the exit status of any child process which already completed + pid, status = os.waitpid(-1, os.WNOHANG) + except OSError: + break + + if pid == 0: + break + + print("Warning -- reap_children() reaped child process %s" + % pid, file=sys.stderr) + environment_altered = True + @contextlib.contextmanager def start_threads(threads, unlock=None): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 1e6b2b54e23..8632837780c 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,12 +1,15 @@ +import contextlib +import errno import importlib +import io +import os import shutil +import socket import stat import sys -import os -import unittest -import socket import tempfile -import errno +import time +import unittest from test import support TESTFN = support.TESTFN @@ -378,6 +381,51 @@ def test_check__all__(self): self.assertRaises(AssertionError, support.check__all__, self, unittest) + @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'), + 'need os.waitpid() and os.WNOHANG') + def test_reap_children(self): + # Make sure that there is no other pending child process + support.reap_children() + + # Create a child process + pid = os.fork() + if pid == 0: + # child process: do nothing, just exit + os._exit(0) + + t0 = time.monotonic() + deadline = time.monotonic() + 60.0 + + was_altered = support.environment_altered + try: + support.environment_altered = False + stderr = io.StringIO() + + while True: + if time.monotonic() > deadline: + self.fail("timeout") + + with contextlib.redirect_stderr(stderr): + support.reap_children() + + # Use environment_altered to check if reap_children() found + # the child process + if support.environment_altered: + break + + # loop until the child process completed + time.sleep(0.100) + + msg = "Warning -- reap_children() reaped child process %s" % pid + self.assertIn(msg, stderr.getvalue()) + self.assertTrue(support.environment_altered) + finally: + support.environment_altered = was_altered + + # Just in case, check again that there is no other + # pending child process + support.reap_children() + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled @@ -398,7 +446,6 @@ def test_check__all__(self): # run_doctest # threading_cleanup # reap_threads - # reap_children # strip_python_stderr # args_from_interpreter_flags # can_symlink

1 0

bpo-31151: Add socketserver.ForkingMixIn.server_close() (#3057)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/aa8ec34ad52bb3b274ce91169e1bc4a598… commit: aa8ec34ad52bb3b274ce91169e1bc4a598655049 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T15:28:16+02:00 summary: bpo-31151: Add socketserver.ForkingMixIn.server_close() (#3057) * Add socketserver.ForkingMixIn.server_close() bpo-31151: socketserver.ForkingMixIn.server_close() now waits until all child processes completed to prevent leaking zombie processes. * Fix test on Windows which doesn't have ForkingMixIn files: A Misc/NEWS.d/next/Library/2017-08-10-13-20-02.bpo-31151.730VBI.rst M Lib/socketserver.py M Lib/test/test_socketserver.py diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 6e1ae9fddda..df171149501 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -547,7 +547,7 @@ class ForkingMixIn: active_children = None max_children = 40 - def collect_children(self): + def collect_children(self, *, blocking=False): """Internal routine to wait for children that have exited.""" if self.active_children is None: return @@ -571,7 +571,8 @@ def collect_children(self): # Now reap all defunct children. for pid in self.active_children.copy(): try: - pid, _ = os.waitpid(pid, os.WNOHANG) + flags = 0 if blocking else os.WNOHANG + pid, _ = os.waitpid(pid, flags) # if the child hasn't exited yet, pid will be 0 and ignored by # discard() below self.active_children.discard(pid) @@ -620,6 +621,10 @@ def process_request(self, request, client_address): finally: os._exit(status) + def server_close(self): + super().server_close() + self.collect_children(blocking=True) + class ThreadingMixIn: """Mix-in class to handle each request in a new thread.""" diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 140a6abf9ef..3d93566607d 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -144,6 +144,10 @@ def run_server(self, svrcls, hdlrbase, testfunc): t.join() server.server_close() self.assertEqual(-1, server.socket.fileno()) + if HAVE_FORKING and isinstance(server, socketserver.ForkingMixIn): + # bpo-31151: Check that ForkingMixIn.server_close() waits until + # all children completed + self.assertFalse(server.active_children) if verbose: print("done") def stream_examine(self, proto, addr): @@ -371,10 +375,7 @@ def wait_done(self): if HAVE_FORKING: class ForkingErrorTestServer(socketserver.ForkingMixIn, BaseErrorTestServer): - def wait_done(self): - [child] = self.active_children - os.waitpid(child, 0) - self.active_children.clear() + pass class SocketWriterTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2017-08-10-13-20-02.bpo-31151.730VBI.rst b/Misc/NEWS.d/next/Library/2017-08-10-13-20-02.bpo-31151.730VBI.rst new file mode 100644 index 00000000000..6eff63653ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-10-13-20-02.bpo-31151.730VBI.rst @@ -0,0 +1,2 @@ +socketserver.ForkingMixIn.server_close() now waits until all child processes +completed to prevent leaking zombie processes.

1 0

bpo-31160: test_tempfile: Fix reap_children() warning (#3056)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/6c8c2943d996b59a48d331f61f22cbe729… commit: 6c8c2943d996b59a48d331f61f22cbe72933910e branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T13:05:06+02:00 summary: bpo-31160: test_tempfile: Fix reap_children() warning (#3056) TestRandomNameSequence.test_process_awareness() now calls os.waitpid() to avoid leaking a zombie process. files: M Lib/test/test_tempfile.py diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index d0cf04b0cb6..710756bde64 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -78,7 +78,6 @@ def setUp(self): def tearDown(self): self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] @@ -184,12 +183,15 @@ def test_process_awareness(self): try: pid = os.fork() if not pid: + # child process os.close(read_fd) os.write(write_fd, next(self.r).encode("ascii")) os.close(write_fd) # bypass the normal exit handlers- leave those to # the parent. os._exit(0) + + # parent process parent_value = next(self.r) child_value = os.read(read_fd, len(parent_value)).decode("ascii") finally: @@ -200,6 +202,10 @@ def test_process_awareness(self): os.kill(pid, signal.SIGKILL) except OSError: pass + + # Read the process exit status to avoid zombie process + os.waitpid(pid, 0) + os.close(read_fd) os.close(write_fd) self.assertNotEqual(child_value, parent_value)

1 0

bpo-31173: Rewrite WSTOPSIG test of test_subprocess (#3055)
by Victor Stinner 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/7b7c6dcfff6a35333988a3c74c895ed19d… commit: 7b7c6dcfff6a35333988a3c74c895ed19dff2e09 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-10T12:37:39+02:00 summary: bpo-31173: Rewrite WSTOPSIG test of test_subprocess (#3055) The current test_child_terminated_in_stopped_state() function test creates a child process which calls ptrace(PTRACE_TRACEME, 0, 0) and then crash (SIGSEGV). The problem is that calling os.waitpid() in the parent process is not enough to close the process: the child process remains alive and so the unit test leaks a child process in a strange state. Closing the child process requires non-trivial code, maybe platform specific. Remove the functional test and replaces it with an unit test which mocks os.waitpid() using a new _testcapi.W_STOPCODE() function to test the WIFSTOPPED() path. files: M Lib/test/test_subprocess.py M Modules/_testcapimodule.c diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d152826213e..1d2d15c7a24 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -29,6 +29,11 @@ except ImportError: threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") @@ -2565,42 +2570,24 @@ def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): proc.communicate(timeout=999) mock_proc_stdin.close.assert_called_once_with() - @unittest.skipIf(not ctypes, 'ctypes module required') - @unittest.skipIf(not sys.executable, 'Test requires sys.executable') - def test_child_terminated_in_stopped_state(self): + @unittest.skipUnless(_testcapi is not None + and hasattr(_testcapi, 'W_STOPCODE'), + 'need _testcapi.W_STOPCODE') + def test_stopped(self): """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" - PTRACE_TRACEME = 0 # From glibc and MacOS (PT_TRACE_ME). - libc_name = ctypes.util.find_library('c') - libc = ctypes.CDLL(libc_name) - if not hasattr(libc, 'ptrace'): - raise unittest.SkipTest('ptrace() required') - - code = textwrap.dedent(f""" - import ctypes - import faulthandler - from test.support import SuppressCrashReport - - libc = ctypes.CDLL({libc_name!r}) - libc.ptrace({PTRACE_TRACEME}, 0, 0) - """) - - child = subprocess.Popen([sys.executable, '-c', code]) - if child.wait() != 0: - raise unittest.SkipTest('ptrace() failed - unable to test') - - code += textwrap.dedent(f""" - with SuppressCrashReport(): - # Crash the process - faulthandler._sigsegv() - """) - child = subprocess.Popen([sys.executable, '-c', code]) - try: - returncode = child.wait() - except: - child.kill() # Clean up the hung stopped process. - raise - self.assertNotEqual(0, returncode) - self.assertLess(returncode, 0) # signal death, likely SIGSEGV. + args = [sys.executable, '-c', 'pass'] + proc = subprocess.Popen(args) + + # Wait until the real process completes to avoid zombie process + pid = proc.pid + pid, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + status = _testcapi.W_STOPCODE(3) + with mock.patch('subprocess.os.waitpid', return_value=(pid, status)): + returncode = proc.wait() + + self.assertEqual(returncode, -3) @unittest.skipUnless(mswindows, "Windows specific tests") diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 655669050dd..95c10184159 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,6 +18,10 @@ # include <winsock2.h> /* struct timeval */ #endif +#ifdef HAVE_SYS_WAIT_H +#include <sys/wait.h> /* For W_STOPCODE */ +#endif + #ifdef WITH_THREAD #include "pythread.h" #endif /* WITH_THREAD */ @@ -4272,6 +4276,7 @@ test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) return _PyObject_FastCallKeywords(func, stack, nargs, kwnames); } + static PyObject* stack_pointer(PyObject *self, PyObject *args) { @@ -4280,6 +4285,20 @@ stack_pointer(PyObject *self, PyObject *args) } +#ifdef W_STOPCODE +static PyObject* +py_w_stopcode(PyObject *self, PyObject *args) +{ + int sig, status; + if (!PyArg_ParseTuple(args, "i", &sig)) { + return NULL; + } + status = W_STOPCODE(sig); + return PyLong_FromLong(status); +} +#endif + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4493,6 +4512,9 @@ static PyMethodDef TestMethods[] = { {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, +#ifdef W_STOPCODE + {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, +#endif {NULL, NULL} /* sentinel */ };

1 0

Daily reference leaks (4243df51fe43): sum=3
by solipsis＠pitrou.net 10 Aug '17

10 Aug '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [-2, 1, 0] memory blocks, sum=-1 test_multiprocessing_forkserver leaked [1, -2, 1] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogV4Zp7U', '--timeout', '7200']

1 0

[3.6] bpo-19903: IDLE: Calltips changed to use inspect.signature (GH-2822) (#3053)
by Terry Jan Reedy 10 Aug '17

10 Aug '17

https://github.com/python/cpython/commit/646f6c3096abfe5bde13f039ebf32bce5b… commit: 646f6c3096abfe5bde13f039ebf32bce5baf083a branch: 3.6 author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-08-10T00:46:29-04:00 summary: [3.6] bpo-19903: IDLE: Calltips changed to use inspect.signature (GH-2822) (#3053) Idlelib.calltips.get_argspec now uses inspect.signature instead of inspect.getfullargspec, like help() does. This improves the signature in the call tip in a few different cases, including builtins converted to provide a signature. A message is added if the object is not callable, has an invalid signature, or if it has positional-only parameters. Patch by Louie Lu.. (cherry picked from commit 3b0f620c1a2a21272a9e2aeca6ca1d1ac10f8162) files: A Misc/NEWS.d/next/IDLE/2017-08-03-14-08-42.bpo-19903.sqE1FS.rst M Lib/idlelib/calltips.py M Lib/idlelib/idle_test/test_calltips.py diff --git a/Lib/idlelib/calltips.py b/Lib/idlelib/calltips.py index a8a3abe6c69..49625eac158 100644 --- a/Lib/idlelib/calltips.py +++ b/Lib/idlelib/calltips.py @@ -123,6 +123,8 @@ def get_entity(expression): _INDENT = ' '*4 # for wrapped signatures _first_param = re.compile(r'(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" +_invalid_method = "invalid method signature" +_argument_positional = "\n['/' marks preceding arguments as positional-only]\n" def get_argspec(ob): @@ -134,25 +136,30 @@ def get_argspec(ob): empty line or _MAX_LINES. For builtins, this typically includes the arguments in addition to the return value. ''' - argspec = "" + argspec = default = "" try: ob_call = ob.__call__ except BaseException: - return argspec - if isinstance(ob, type): - fob = ob.__init__ - elif isinstance(ob_call, types.MethodType): - fob = ob_call - else: - fob = ob - if isinstance(fob, (types.FunctionType, types.MethodType)): - argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) - if (isinstance(ob, (type, types.MethodType)) or - isinstance(ob_call, types.MethodType)): - argspec = _first_param.sub("", argspec) + return default + + fob = ob_call if isinstance(ob_call, types.MethodType) else ob + + try: + argspec = str(inspect.signature(fob)) + except ValueError as err: + msg = str(err) + if msg.startswith(_invalid_method): + return _invalid_method + + if '/' in argspec: + """Using AC's positional argument should add the explain""" + argspec += _argument_positional + if isinstance(fob, type) and argspec == '()': + """fob with no argument, use default callable argspec""" + argspec = _default_callable_argspec lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) - if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + if len(argspec) > _MAX_COLS else [argspec] if argspec else []) if isinstance(ob_call, types.MethodType): doc = ob_call.__doc__ @@ -171,6 +178,7 @@ def get_argspec(ob): argspec = _default_callable_argspec return argspec + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_calltips', verbosity=2) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 97e15fe10e0..f6b71306ac8 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -46,6 +46,7 @@ def test_builtins(self): # Python class that inherits builtin methods class List(list): "List() doc" + # Simulate builtin with no docstring for default tip test class SB: __call__ = None @@ -53,18 +54,29 @@ def gtest(obj, out): self.assertEqual(signature(obj), out) if List.__doc__ is not None: - gtest(List, List.__doc__) + gtest(List, List.__doc__) # This and append_doc changed in 3.7. gtest(list.__new__, - 'Create and return a new object. See help(type) for accurate signature.') + '(*args, **kwargs)\nCreate and return a new object.' + ' See help(type) for accurate signature.') gtest(list.__init__, + '(self, /, *args, **kwargs)' + ct._argument_positional + '\n' + 'Initialize self. See help(type(self)) for accurate signature.') - append_doc = "L.append(object) -> None -- append object to end" #see3.7 + + append_doc = "L.append(object) -> None -- append object to end" gtest(list.append, append_doc) gtest([].append, append_doc) gtest(List.append, append_doc) gtest(types.MethodType, "method(function, instance)") gtest(SB(), default_tip) + import re + p = re.compile('') + gtest(re.sub, '''(pattern, repl, string, count=0, flags=0)\nReturn the string obtained by replacing the leftmost +non-overlapping occurrences of the pattern in string by the +replacement repl. repl can be either a string or a callable; +if a string, backslash escapes in it are processed. If it is +a callable, it's passed the match object and must return''') + gtest(p.sub, '''(repl, string, count=0)\nReturn the string obtained by replacing the leftmost non-overlapping occurrences o...''') def test_signature_wrap(self): if textwrap.TextWrapper.__doc__ is not None: @@ -132,12 +144,20 @@ def test_starred_parameter(self): # test that starred first parameter is *not* removed from argspec class C: def m1(*args): pass - def m2(**kwds): pass c = C() - for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"), - (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),): + for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"),): self.assertEqual(signature(meth), mtip) + def test_invalid_method_signature(self): + class C: + def m2(**kwargs): pass + class Test: + def __call__(*, a): pass + + mtip = ct._invalid_method + self.assertEqual(signature(C().m2), mtip) + self.assertEqual(signature(Test()), mtip) + def test_non_ascii_name(self): # test that re works to delete a first parameter name that # includes non-ascii chars, such as various forms of A. @@ -156,17 +176,23 @@ def test_attribute_exception(self): class NoCall: def __getattr__(self, name): raise BaseException - class Call(NoCall): + class CallA(NoCall): + def __call__(oui, a, b, c): + pass + class CallB(NoCall): def __call__(self, ci): pass - for meth, mtip in ((NoCall, default_tip), (Call, default_tip), - (NoCall(), ''), (Call(), '(ci)')): + + for meth, mtip in ((NoCall, default_tip), (CallA, default_tip), + (NoCall(), ''), (CallA(), '(a, b, c)'), + (CallB(), '(ci)')): self.assertEqual(signature(meth), mtip) def test_non_callables(self): for obj in (0, 0.0, '0', b'0', [], {}): self.assertEqual(signature(obj), '') + class Get_entityTest(unittest.TestCase): def test_bad_entity(self): self.assertIsNone(ct.get_entity('1/0')) diff --git a/Misc/NEWS.d/next/IDLE/2017-08-03-14-08-42.bpo-19903.sqE1FS.rst b/Misc/NEWS.d/next/IDLE/2017-08-03-14-08-42.bpo-19903.sqE1FS.rst new file mode 100644 index 00000000000..f25fc80c3dd --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-03-14-08-42.bpo-19903.sqE1FS.rst @@ -0,0 +1,3 @@ +IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +This improves calltips for builtins converted to use Argument Clinic. +Patch by Louie Lu.

1 0