[Python-checkins] bpo-38546: multiprocessing tests stop the resource tracker (GH-17641)
Victor Stinner
webhook-mailer at python.org
Tue Dec 17 12:37:47 EST 2019
https://github.com/python/cpython/commit/9707e8e22d80ca97bf7a9812816701cecde6d226
commit: 9707e8e22d80ca97bf7a9812816701cecde6d226
branch: master
author: Victor Stinner <vstinner at python.org>
committer: GitHub <noreply at github.com>
date: 2019-12-17T18:37:26+01:00
summary:
bpo-38546: multiprocessing tests stop the resource tracker (GH-17641)
Multiprocessing and concurrent.futures tests now stop the resource
tracker process when tests complete.
Add ResourceTracker._stop() method to
multiprocessing.resource_tracker.
Add _cleanup_tests() helper function to multiprocessing.util: share
code between multiprocessing and concurrent.futures tests.
files:
A Misc/NEWS.d/next/Tests/2019-12-17-15-27-07.bpo-38546.82JwN2.rst
M Lib/multiprocessing/resource_tracker.py
M Lib/multiprocessing/util.py
M Lib/test/_test_multiprocessing.py
M Lib/test/test_concurrent_futures.py
diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py
index 61a6dd66e72e6..c9bfa9b82b6e6 100644
--- a/Lib/multiprocessing/resource_tracker.py
+++ b/Lib/multiprocessing/resource_tracker.py
@@ -50,6 +50,19 @@ def __init__(self):
self._fd = None
self._pid = None
+ def _stop(self):
+ with self._lock:
+ if self._fd is None:
+ # not running
+ return
+
+ # closing the "alive" file descriptor stops main()
+ os.close(self._fd)
+ self._fd = None
+
+ os.waitpid(self._pid, 0)
+ self._pid = None
+
def getfd(self):
self.ensure_running()
return self._fd
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index 3e640b944e958..4bc7782c00c15 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -439,3 +439,28 @@ def close_fds(*fds):
"""Close each file descriptor given as an argument"""
for fd in fds:
os.close(fd)
+
+
+def _cleanup_tests():
+ """Cleanup multiprocessing resources when multiprocessing tests
+ completed."""
+
+ from test import support
+
+ # cleanup multiprocessing
+ process._cleanup()
+
+ # Stop the ForkServer process if it's running
+ from multiprocessing import forkserver
+ forkserver._forkserver._stop()
+
+ # Stop the ResourceTracker process if it's running
+ from multiprocessing import resource_tracker
+ resource_tracker._resource_tracker._stop()
+
+ # bpo-37421: Explicitly call _run_finalizers() to remove immediately
+ # temporary directories created by multiprocessing.util.get_temp_dir().
+ _run_finalizers()
+ support.gc_collect()
+
+ support.reap_children()
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index c0da7d8912e64..4e48cd45e14ca 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -5695,16 +5695,7 @@ def tearDownModule():
if need_sleep:
time.sleep(0.5)
- multiprocessing.process._cleanup()
-
- # Stop the ForkServer process if it's running
- from multiprocessing import forkserver
- forkserver._forkserver._stop()
-
- # bpo-37421: Explicitly call _run_finalizers() to remove immediately
- # temporary directories created by multiprocessing.util.get_temp_dir().
- multiprocessing.util._run_finalizers()
- test.support.gc_collect()
+ multiprocessing.util._cleanup_tests()
remote_globs['setUpModule'] = setUpModule
remote_globs['tearDownModule'] = tearDownModule
diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py
index 8b7518530193d..7b10f81ff207c 100644
--- a/Lib/test/test_concurrent_futures.py
+++ b/Lib/test/test_concurrent_futures.py
@@ -1306,17 +1306,7 @@ def setUpModule():
def tearDownModule():
support.threading_cleanup(*_threads_key)
- support.reap_children()
-
- # cleanup multiprocessing
- multiprocessing.process._cleanup()
- # Stop the ForkServer process if it's running
- from multiprocessing import forkserver
- forkserver._forkserver._stop()
- # bpo-37421: Explicitly call _run_finalizers() to remove immediately
- # temporary directories created by multiprocessing.util.get_temp_dir().
- multiprocessing.util._run_finalizers()
- support.gc_collect()
+ multiprocessing.util._cleanup_tests()
if __name__ == "__main__":
diff --git a/Misc/NEWS.d/next/Tests/2019-12-17-15-27-07.bpo-38546.82JwN2.rst b/Misc/NEWS.d/next/Tests/2019-12-17-15-27-07.bpo-38546.82JwN2.rst
new file mode 100644
index 0000000000000..78d9df3469182
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-12-17-15-27-07.bpo-38546.82JwN2.rst
@@ -0,0 +1,2 @@
+Multiprocessing and concurrent.futures tests now stop the resource tracker
+process when tests complete.
More information about the Python-checkins
mailing list