[Python-checkins] bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)

Miss Islington (bot) webhook-mailer at python.org
Fri Nov 23 12:41:59 EST 2018


https://github.com/python/cpython/commit/903a3e8d67b61594c0fa17fb201769ca924b38f8
commit: 903a3e8d67b61594c0fa17fb201769ca924b38f8
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2018-11-23T09:41:54-08:00
summary:

bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)


Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform

https://bugs.python.org/issue29877
(cherry picked from commit 1d817e4c8259f49602eefe9729743f6d9d748e8d)

Co-authored-by: Dustin Spicuzza <dustin at virtualroadside.com>

files:
A Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
M Lib/compileall.py
M Lib/test/test_compileall.py

diff --git a/Lib/compileall.py b/Lib/compileall.py
index 72592126d74c..40b148d2017e 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -16,10 +16,6 @@
 import py_compile
 import struct
 
-try:
-    from concurrent.futures import ProcessPoolExecutor
-except ImportError:
-    ProcessPoolExecutor = None
 from functools import partial
 
 __all__ = ["compile_dir","compile_file","compile_path"]
@@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     workers:   maximum number of parallel workers
     invalidation_mode: how the up-to-dateness of the pyc will be checked
     """
-    if workers is not None and workers < 0:
-        raise ValueError('workers must be greater or equal to 0')
-
+    ProcessPoolExecutor = None
+    if workers is not None:
+        if workers < 0:
+            raise ValueError('workers must be greater or equal to 0')
+        elif workers != 1:
+            try:
+                # Only import when needed, as low resource platforms may
+                # fail to import it
+                from concurrent.futures import ProcessPoolExecutor
+            except ImportError:
+                workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 38d7b996c7c2..2995e08aa8b5 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -161,7 +161,7 @@ def test_compile_dir_pathlike(self):
         self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
         self.assertTrue(os.path.isfile(self.bc_path))
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_pool_called(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(pool_mock.called)
@@ -171,19 +171,19 @@ def test_compile_workers_non_positive(self):
                                     "workers must be greater or equal to 0"):
             compileall.compile_dir(self.directory, workers=-1)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_workers_cpu_count(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=0)
         self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     @mock.patch('compileall.compile_file')
     def test_compile_one_worker(self, compile_file_mock, pool_mock):
         compileall.compile_dir(self.directory, quiet=True)
         self.assertFalse(pool_mock.called)
         self.assertTrue(compile_file_mock.called)
 
-    @mock.patch('compileall.ProcessPoolExecutor', new=None)
+    @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
     @mock.patch('compileall.compile_file')
     def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
diff --git a/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
new file mode 100644
index 000000000000..cc09533b7124
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
@@ -0,0 +1,2 @@
+compileall: import ProcessPoolExecutor only when needed, preventing hangs on
+low resource platforms



More information about the Python-checkins mailing list