[Python-checkins] cpython (merge 3.2 -> default): Issue #15320: Make iterating the list of tests thread-safe when running tests

[antoine.pitrou]

http://hg.python.org/cpython/rev/43ae2a243eca
changeset:   78285:43ae2a243eca
parent:      78283:a9a1d3d6d0dc
parent:      78284:d7a64e095930
user:        Antoine Pitrou <solipsis at pitrou.net>
date:        Thu Jul 26 00:47:15 2012 +0200
summary:
  Issue #15320: Make iterating the list of tests thread-safe when running tests in multiprocess mode.
Patch by Chris Jerdonek.

files:
  Lib/test/regrtest.py |  42 ++++++++++++++++++++++---------
  Misc/NEWS            |   3 ++
  2 files changed, 32 insertions(+), 13 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -614,17 +614,7 @@
         from subprocess import Popen, PIPE
         debug_output_pat = re.compile(r"\[\d+ refs\]$")
         output = Queue()
-        def tests_and_args():
-            for test in tests:
-                args_tuple = (
-                    (test, verbose, quiet),
-                    dict(huntrleaks=huntrleaks, use_resources=use_resources,
-                         debug=debug, output_on_failure=verbose3,
-                         timeout=timeout, failfast=failfast,
-                         match_tests=match_tests)
-                )
-                yield (test, args_tuple)
-        pending = tests_and_args()
+        pending = MultiprocessTests(tests)
         opt_args = support.args_from_interpreter_flags()
         base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest']
         def work():
@@ -632,10 +622,17 @@
             try:
                 while True:
                     try:
-                        test, args_tuple = next(pending)
+                        test = next(pending)
                     except StopIteration:
                         output.put((None, None, None, None))
                         return
+                    args_tuple = (
+                        (test, verbose, quiet),
+                        dict(huntrleaks=huntrleaks, use_resources=use_resources,
+                             debug=debug, output_on_failure=verbose3,
+                             timeout=timeout, failfast=failfast,
+                             match_tests=match_tests)
+                    )
                     # -E is needed by some tests, e.g. test_import
                     # Running the child from the same working directory ensures
                     # that TEMPDIR for the child is the same when
@@ -694,7 +691,7 @@
                 test_index += 1
         except KeyboardInterrupt:
             interrupted = True
-            pending.close()
+            pending.interrupted = True
         for worker in workers:
             worker.join()
     else:
@@ -840,6 +837,25 @@
             tests.append(mod)
     return stdtests + sorted(tests)
 
+# We do not use a generator so multiple threads can call next().
+class MultiprocessTests(object):
+
+    """A thread-safe iterator over tests for multiprocess mode."""
+
+    def __init__(self, tests):
+        self.interrupted = False
+        self.lock = threading.Lock()
+        self.tests = tests
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        with self.lock:
+            if self.interrupted:
+                raise StopIteration('tests interrupted')
+            return next(self.tests)
+
 def replace_stdout():
     """Set stdout encoder error handler to backslashreplace (as stderr error
     handler) to avoid UnicodeEncodeError when printing a traceback"""
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -211,6 +211,9 @@
 Tests
 -----
 
+- Issue #15320: Make iterating the list of tests thread-safe when running
+  tests in multiprocess mode. Patch by Chris Jerdonek.
+
 - Issue #15168: Move importlib.test to test.test_importlib.
 
 - Issue #15091: Reactivate a test on UNIX which was failing thanks to a

-- 
Repository URL: http://hg.python.org/cpython